def test_multiple_nested_groups(self):
""" Test the receiving of multiple nested groups """
parser = FIXParser(self.receiver,
debug=True,
group_fields={100: [101, 102, 200],
200: [201, 202], },
header_fields=[8, 9])
parser.on_data_received(to_fix('8=FIX.4.2',
'9=60',
'100=2',
'101=a',
'102=b',
'200=2',
'201=abc',
'202=def',
'201=zzz',
'101=c',
'102=d',
'10=002'))
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
print message.store
self.assertIsNotNone(message)
self.assertEquals(4, len(message))
self.assertTrue(100 in message)
self.assertEquals(2, len(message[100]))
group = message[100]
self.assertIsNotNone(group)
self.assertEquals(2, len(group))
self.assertEquals(3, len(group[0]))
self.assertTrue(101 in group[0])
self.assertTrue(102 in group[0])
self.assertTrue(200 in group[0])
self.assertEquals('a', group[0][101])
self.assertEquals('b', group[0][102])
self.assertEquals(2, len(group[1]))
self.assertTrue(101 in group[1])
self.assertTrue(102 in group[1])
self.assertTrue(200 not in group[1])
self.assertEquals('c', group[1][101])
self.assertEquals('d', group[1][102])
subgroup = group[0]
self.assertIsNotNone(subgroup)
self.assertEquals(3, len(subgroup))
self.assertEquals(2, len(subgroup[200]))
subgroup200 = subgroup[200]
self.assertEquals(2, len(subgroup200[0]))
self.assertTrue(201 in subgroup200[0])
self.assertTrue(202 in subgroup200[0])
self.assertEquals('abc', subgroup200[0][201])
self.assertEquals('def', subgroup200[0][202])
self.assertEquals(1, len(subgroup200[1]))
self.assertTrue(201 in subgroup200[1])
self.assertTrue(202 not in subgroup200[1])
self.assertEquals('zzz', subgroup200[1][201])
def test_parse_field(self):
""" Basic _parse_field function test """
parser = FIXParser(self.receiver)
field = parser._parse_field('8=a')
self.assertEquals(8, field[0])
self.assertEquals('a', field[1])
def test_simple_group_fields(self):
""" Simple group field testing. """
parser = FIXParser(self.receiver,
group_fields={100: [101, 102, 200],
200: [201, 202], },
header_fields=[8, 9])
parser.on_data_received(to_fix('8=FIX.4.2',
'9=18',
'100=1',
'101=a',
'102=b',
'10=099'))
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(4, len(message))
self.assertTrue(100 in message)
self.assertEquals(1, len(message[100]))
group = message[100]
self.assertIsNotNone(group)
self.assertEquals(1, len(group))
self.assertEquals(2, len(group[0]))
self.assertTrue(101 in group[0])
self.assertTrue(102 in group[0])
self.assertEquals('a', group[0][101])
self.assertEquals('b', group[0][102])
def test_one_byte_at_a_time(self):
""" Receive a message split up into single bytes """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
text = to_fix('8=FIX.4.2',
'9=23',
'35=A',
'919=this',
'955=that',
'10=013')
for c in text:
parser.on_data_received(c)
self.assertFalse(parser.is_parsing)
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(6, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '23'),
(35, 'A'),
(919, 'this'),
(955, 'that'),
(10, '013')]))
def test_simple_message(self):
""" Basic function test. """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
self.assertFalse(parser.is_parsing)
# message taken from wikipedia article
parser.on_data_received(to_fix('8=FIX.4.2',
'9=65',
'35=A',
'49=SERVER',
'56=CLIENT',
'34=177',
'52=20090107-18:15:16',
'98=0',
'108=30',
'10=062'))
self.assertFalse(parser.is_parsing)
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(10, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '65'),
(35, 'A'),
(10, '062')]))
def test_multiple_message(self):
""" Receive two messages in one data buffer """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
self.assertFalse(parser.is_parsing)
parser.on_data_received(to_fix('8=FIX.4.2',
'9=5',
'35=A',
'10=178',
'8=FIX.4.2',
'9=17',
'35=E',
'99=forsooth',
'10=013'))
self.assertFalse(parser.is_parsing)
self.assertEquals(2, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(5, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '17'),
(35, 'E'),
(99, 'forsooth'),
(10, '013')]))
def test_message_too_large(self):
""" Test for too long message """
parser = FIXParser(self.receiver,
header_fields=[8, 9],
max_length=100)
with self.assertRaises(FIXLengthTooLongError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'42=A' + 'BB'*100))
def __init__(self, name, transport, **kwargs):
""" FIXProtocol initialization
Args:
name: A descriptive name given to this connection.
transport: The transport used to interface with the
networking layer.
config: A dict() for the endpoint configuration.
See the ROLES section in sample_config.py for examples.
link_config: A dict for the link configuration.
See the CONNECTIONS section in sample_config.py for
examples and documentation.
debug: Set this to True for debug logging. (Default: False)
Raises:
ValueError: name and queue are required parameters.
"""
if name is None:
raise ValueError("name is None")
if transport is None:
raise ValueError("name is None")
self.name = name
self.transport = transport
self.config = kwargs.get('config', dict())
self.link_config = kwargs.get('link_config', dict())
self._debug = kwargs.get('debug')
if len(self.link_config.get('protocol_version', '')) == 0:
raise ValueError('link_config missing protocol_version')
# heartbeat processing
self.heartbeat = self.link_config.get('heartbeat', 0)
self.filter_heartbeat = False
self._testrequest_id = None
self._testrequest_time = None
# protocol state information
self._send_seqno = self.link_config.get('send_seqno', 0)
self._last_send_time = datetime.datetime.now()
self._received_seqno = 0
self._last_received_time = datetime.datetime.now()
self._parser = FIXParser(self,
header_fields=self.link_config.get(
'header_fields', None),
binary_fields=self.link_config.get(
'binary_fields', None),
group_fields=self.link_config.get(
'group_fields', None),
max_length=self.link_config.get(
'max_length', 2048))
self._logger = logging.getLogger(__name__)
def test_partial_message(self):
""" Test for partial input. """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
self.assertFalse(parser.is_parsing)
self.assertEquals(0, self.receiver.count)
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'35=A'))
self.assertEquals(0, self.receiver.count)
self.assertTrue(parser.is_parsing)
def test_parser_reset(self):
""" Test that the parser resets on an error """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
# Tag ID is not a number
self.assertFalse(parser.is_parsing)
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'abcd=A'))
self.assertFalse(parser.is_parsing)
def test_bad_binary_fields(self):
""" Test bad binary fields """
parser = FIXParser(self.receiver,
binary_fields=[1000],
header_fields=[8, 9])
# Missing binary value portion of binary field
# BUGBUG: This can cause some problems, because the parser
# does not attempt to validate until the entire
# field has been read in. Which this will fail because
# the length goes past the end of the message.
# For now, live with this.
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'1000=5',
'999=11',
'10=001'))
# Missing binary length portion (the only time this
# really impacts something is if the field has an
# embedded \x01).
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'1001=1010\x011010',
'10=001'))
parser.on_data_received(to_fix('8=FIX.4.2',
'9=14',
'1001=10101010',
'10=127'))
self.assertIsNone(self.receiver.last_error)
self.assertEquals(1, self.receiver.count)
self.assertIsNotNone(self.receiver.last_received_message)
def test_message_binary_too_long(self):
""" Test for message with missing binary data """
parser = FIXParser(self.receiver,
header_fields=[8, 9],
binary_fields=[1000],
max_length=100)
# length too short
with self.assertRaises(FIXLengthTooLongError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'42=A',
'1000=128',
'1001=abababababab',
'10=000'))
def test_grouped_binary_fields(self):
""" Test binary fields that are in a group. """
parser = FIXParser(self.receiver,
debug=True,
group_fields={200: [201, 202, 99, 100]},
binary_fields=[99],
header_fields=[8, 9])
text = to_fix('8=FIX.4.2',
'9=80',
'35=A',
'200=2',
'201=aabc',
'99=5',
'100=abcde',
'201=zzzaa',
'202=myname',
'99=5',
'100=zztop',
'955=that',
'10=201')
parser.on_data_received(text)
self.assertFalse(parser.is_parsing)
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(6, len(message))
self.assertEquals(2, len(message[200]))
self.assertTrue(200 in message)
self.assertTrue(955 in message)
subgroup = message[200][0]
self.assertEquals(3, len(subgroup))
self.assertTrue(201 in subgroup)
self.assertTrue(99 in subgroup)
self.assertTrue(100 in subgroup)
subgroup = message[200][1]
self.assertEquals(4, len(subgroup))
self.assertTrue(201 in subgroup)
self.assertTrue(202 in subgroup)
self.assertTrue(99 in subgroup)
self.assertTrue(100 in subgroup)
def test_parse_field_bad_input(self):
""" Test bad _parse_field inputs """
parser = FIXParser(self.receiver)
# missing '='
with self.assertRaises(FIXParserError):
parser._parse_field('abcde')
# bad tag id
with self.assertRaises(FIXParserError):
parser._parse_field('a=a')
# missing tag id
with self.assertRaises(FIXParserError):
parser._parse_field('=a')
# bad tag id
with self.assertRaises(FIXParserError):
parser._parse_field('10b=a')
def test_header_fields(self):
""" Header field testing. """
parser = FIXParser(self.receiver,
header_fields=[8, 9, 320])
parser.on_data_received(to_fix('8=FIX.4.2',
'9=5',
'35=A',
'10=178'))
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
message[320] = 'hello there'
self.assertEquals(5, len(message))
# verify the order of the message
items = [(k, v) for k, v in message.items()]
self.assertEquals(8, items[0][0])
self.assertEquals(9, items[1][0])
self.assertEquals(320, items[2][0])
def test_partial_binary_data(self):
""" Receive a piece of binary data split into two parts """
parser = FIXParser(self.receiver,
binary_fields=[99],
header_fields=[8, 9])
text = to_fix('8=FIX.4.2',
'9=38',
'35=A') + '99=5\x01100=12'
text2 = to_fix('345',
'919=this',
'955=that',
'10=198')
parser.on_data_received(text)
self.assertTrue(parser.is_parsing)
parser.on_data_received(text2)
self.assertFalse(parser.is_parsing)
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(8, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '38'),
(35, 'A'),
(99, '5'),
(100, '12345'),
(919, 'this'),
(955, 'that'),
(10, '198')]))
def test_message_bad_binary_length(self):
""" Test for message with missing binary data """
parser = FIXParser(self.receiver,
header_fields=[8, 9],
binary_fields=[1000],
max_length=100)
# length too short
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'42=A',
'1000=2',
'1001=abababababab',
'10=000'))
# length too long
# This will not raise an error
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'42=A',
'1000=20',
'1001=ab',
'10=000'))
self.assertEquals(0, self.receiver.count)
self.assertTrue(parser.is_parsing)
def test_multiple_groups(self):
""" Test the receiving of multiple groups """
parser = FIXParser(self.receiver,
group_fields={100: [101, 102, 200],
200: [201, 202], },
header_fields=[8, 9])
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'100=2',
'101=a',
'102=b',
'101=aa',
'102=bb',
'10=135'))
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(4, len(message))
self.assertTrue(100 in message)
self.assertEquals(2, len(message[100]))
group = message[100]
self.assertIsNotNone(group)
self.assertEquals(2, len(group))
self.assertEquals(2, len(group[0]))
self.assertTrue(101 in group[0])
self.assertTrue(102 in group[0])
self.assertEquals('a', group[0][101])
self.assertEquals('b', group[0][102])
self.assertEquals(2, len(group[1]))
self.assertTrue(101 in group[1])
self.assertTrue(102 in group[1])
self.assertEquals('aa', group[1][101])
self.assertEquals('bb', group[1][102])
def test_bad_syntax(self):
""" Test for various bad syntax cases """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
# Tag ID is not a number
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'abcd=A'))
# Missing '=' and value portion
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4',
'9=32',
'35'))
# Missing tag ID portion
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4',
'9=32',
'=A'))
def test_message_starts_incorrectly(self):
""" Message must start with tag 8 """
parser = FIXParser(self.receiver,
header_fields=[8, 9])
# message does not start with tag 8
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('18=FIX.4.2',
'9=32',
'8=FIX.4.2',
'35=A',
'10=100'))
# unexpected tag 8
with self.assertRaises(FIXParserError):
parser.on_data_received(to_fix('8=FIX.4.2',
'9=32',
'8=abcdef',
'35=A',
'10=100'))
def test_binary_fields(self):
""" Binary field testing. """
parser = FIXParser(self.receiver,
binary_fields=[1000, 1010],
header_fields=[8, 9])
# Test with embedded binary \x01
parser.on_data_received(to_fix('8=FIX.4.2',
'9=18',
'1000=5',
'1001=\x01\x02\x03\x04\x05',
'10=066'))
self.assertEquals(1, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(5, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '18'),
(1000, '5'),
(1001, '\x01\x02\x03\x04\x05'),
(10, '066')]))
# Test with embedded '=' signs
parser.on_data_received(to_fix('8=FIX.4.2',
'9=18',
'1000=5',
'1001=31=20',
'10=054'))
self.assertEquals(2, self.receiver.count)
message = self.receiver.last_received_message
self.assertIsNotNone(message)
self.assertEquals(5, len(message))
self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'),
(9, '18'),
(1000, '5'),
(1001, '31=20'),
(10, '054')]))
class FIXProtocol(object):
""" Implements the protocol interface. Instead of dealing directly
with byte streams, we now deal with Messages. This layer is
responsible for most of the administration level logic, such as
handling of heartbeats/TestRequests. This class will also
handle tracking of certain appliction counters, such as
message sequence numbers.
Attributes:
name: A descriptive name given to this connection.
transport: The transport interface for this connection.
config: A dict() for the endpoint configuration.
See the ROLES section in sample_config.py for examples.
link_config: A dict for the link configuration.
See the CONNECTIONS section in sample_config.py for
examples and documentation.
heartbeat: The heartbeat interval (in secs). If set to 0,
then no heartbeat processing is performed. (Default:0)
filter_heartbeat: If this is set to True, then heartbeat and
TestRequest messages will not be passed on via
on_message_received(). (Default: False)
debug: Set this to True for debug logging. (Default: False)
"""
# pylint: disable=too-many-instance-attributes
def __init__(self, name, transport, **kwargs):
""" FIXProtocol initialization
Args:
name: A descriptive name given to this connection.
transport: The transport used to interface with the
networking layer.
config: A dict() for the endpoint configuration.
See the ROLES section in sample_config.py for examples.
link_config: A dict for the link configuration.
See the CONNECTIONS section in sample_config.py for
examples and documentation.
debug: Set this to True for debug logging. (Default: False)
Raises:
ValueError: name and queue are required parameters.
"""
if name is None:
raise ValueError("name is None")
if transport is None:
raise ValueError("name is None")
self.name = name
self.transport = transport
self.config = kwargs.get('config', dict())
self.link_config = kwargs.get('link_config', dict())
self._debug = kwargs.get('debug')
if len(self.link_config.get('protocol_version', '')) == 0:
raise ValueError('link_config missing protocol_version')
# heartbeat processing
self.heartbeat = self.link_config.get('heartbeat', 0)
self.filter_heartbeat = False
self._testrequest_id = None
self._testrequest_time = None
# protocol state information
self._send_seqno = self.link_config.get('send_seqno', 0)
self._last_send_time = datetime.datetime.now()
self._received_seqno = 0
self._last_received_time = datetime.datetime.now()
self._parser = FIXParser(self,
header_fields=self.link_config.get(
'header_fields', None),
binary_fields=self.link_config.get(
'binary_fields', None),
group_fields=self.link_config.get(
'group_fields', None),
max_length=self.link_config.get(
'max_length', 2048))
self._logger = logging.getLogger(__name__)
def on_data_received(self, data):
""" This is the notification from the transport.
"""
# pass this onto the parser
self._parser.on_data_received(data)
def prepare_send_message(self, message):
""" Sends a message via the transport.
"""
self._send_seqno += 1
send_time = datetime.datetime.now()
# update some of the required fields for sending
message[8] = self.link_config['protocol_version']
message[34] = self._send_seqno
message[52] = format_time(send_time)
# apply any common fields
if 'common_fields' in self.link_config:
for tag, value in self.link_config['common_fields']:
message[tag] = value
# verify required tags
for tag in self.link_config['required_fields']:
# these two tags are updated when generating the binary
if tag in {9, 10}:
continue
if tag not in message or len(str(message[tag])) == 0:
raise FIXDataError(tag, 'missing field: id:{0}'.format(tag))
self._last_send_time = send_time
return message
def on_message_received(self, message, message_length, checksum):
""" This is the callback from the parser when a message has
been received.
"""
# verify required tags
for tag in self.link_config['required_fields']:
if tag not in message or len(str(message[tag])) == 0:
raise FIXDataError(tag, 'missing field: id:{0}'.format(tag))
# verify the protocol version
if 'protocol_version' in self.link_config:
if self.link_config['protocol_version'] != message[8]:
raise FIXDataError(
8, 'version mismatch: expect:{0} received:{1}'.format(
self.link_config['protocol_version'],
message[8]
))
# verify the length and checksum
if message_length != int(message[9]):
raise FIXDataError(
9, 'length mismatch: expect:{0} received:{1}'.format(
message_length, int(message[9])))
if checksum != int(message[10]):
raise FIXDataError(
10, 'checksum mismatch: expect:{0} received:{1}'.format(
checksum, message[10]))
self._last_received_time = datetime.datetime.now()
# Have we received our testrequest response?
if (message.msg_type() == FIX.HEARTBEAT and
message.get(112, '') == self._testrequest_id):
self._testrequest_time = None
self._testrequest_id = None
# We have received a testrequest and need to send a response
if message.msg_type() == FIX.TEST_REQUEST:
self.transport.send_message(
FIXMessage(source=[(35, FIX.HEARTBEAT),
(112, message[112]),
(49, self.link_config['sender_compid']),
(56, self.link_config['target_compid'])]))
if (not self.filter_heartbeat or
(message.msg_type() not in {FIX.HEARTBEAT, FIX.TEST_REQUEST})):
self.transport.on_message_received(message)
def on_error_received(self, error):
""" This is the callback from the parser when an error in the
message has been detected.
"""
# pylint: disable=no-self-use
raise error
def on_timer_tick_received(self):
""" This is the Twisted timer callback when the heartbeat interval
has elapsed.
"""
if self.heartbeat <= 0:
return
now = datetime.datetime.now()
# Have we received a testrequest response before we timed out?
if (self._testrequest_id is not None and
(now - self._testrequest_time).seconds > 2*self.heartbeat):
raise FIXTimeoutError('testrequest response timeout')
# if heartbeat seconds/2 have elapsed since the last time
# a message was sent, send a heartbeat
if (now - self._last_send_time).seconds > (self.heartbeat/2):
self.transport.send_message(
FIXMessage(source=[(35, FIX.HEARTBEAT),
(49, self.link_config['sender_compid']),
(56, self.link_config['target_compid'])]))
# if heartbeat seconds + "some transmission time" have elapsed
# since a message was received, send a TestRequest
if (now - self._last_received_time).seconds > self.heartbeat:
testrequest_id = "TR{0}".format(format_time(now))
self._testrequest_time = now
self._testrequest_id = testrequest_id
self.transport.send_message(
FIXMessage(source=[(35, FIX.TEST_REQUEST),
(112, testrequest_id),
(49, self.link_config['sender_compid']),
(56, self.link_config['target_compid'])]))
