def test_multiple_nested_groups(self): """ Test the receiving of multiple nested groups """ parser = FIXParser(self.receiver, debug=True, group_fields={100: [101, 102, 200], 200: [201, 202], }, header_fields=[8, 9]) parser.on_data_received(to_fix('8=FIX.4.2', '9=60', '100=2', '101=a', '102=b', '200=2', '201=abc', '202=def', '201=zzz', '101=c', '102=d', '10=002')) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message print message.store self.assertIsNotNone(message) self.assertEquals(4, len(message)) self.assertTrue(100 in message) self.assertEquals(2, len(message[100])) group = message[100] self.assertIsNotNone(group) self.assertEquals(2, len(group)) self.assertEquals(3, len(group[0])) self.assertTrue(101 in group[0]) self.assertTrue(102 in group[0]) self.assertTrue(200 in group[0]) self.assertEquals('a', group[0][101]) self.assertEquals('b', group[0][102]) self.assertEquals(2, len(group[1])) self.assertTrue(101 in group[1]) self.assertTrue(102 in group[1]) self.assertTrue(200 not in group[1]) self.assertEquals('c', group[1][101]) self.assertEquals('d', group[1][102]) subgroup = group[0] self.assertIsNotNone(subgroup) self.assertEquals(3, len(subgroup)) self.assertEquals(2, len(subgroup[200])) subgroup200 = subgroup[200] self.assertEquals(2, len(subgroup200[0])) self.assertTrue(201 in subgroup200[0]) self.assertTrue(202 in subgroup200[0]) self.assertEquals('abc', subgroup200[0][201]) self.assertEquals('def', subgroup200[0][202]) self.assertEquals(1, len(subgroup200[1])) self.assertTrue(201 in subgroup200[1]) self.assertTrue(202 not in subgroup200[1]) self.assertEquals('zzz', subgroup200[1][201])
def test_parse_field(self): """ Basic _parse_field function test """ parser = FIXParser(self.receiver) field = parser._parse_field('8=a') self.assertEquals(8, field[0]) self.assertEquals('a', field[1])
def test_simple_group_fields(self): """ Simple group field testing. """ parser = FIXParser(self.receiver, group_fields={100: [101, 102, 200], 200: [201, 202], }, header_fields=[8, 9]) parser.on_data_received(to_fix('8=FIX.4.2', '9=18', '100=1', '101=a', '102=b', '10=099')) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(4, len(message)) self.assertTrue(100 in message) self.assertEquals(1, len(message[100])) group = message[100] self.assertIsNotNone(group) self.assertEquals(1, len(group)) self.assertEquals(2, len(group[0])) self.assertTrue(101 in group[0]) self.assertTrue(102 in group[0]) self.assertEquals('a', group[0][101]) self.assertEquals('b', group[0][102])
def test_one_byte_at_a_time(self): """ Receive a message split up into single bytes """ parser = FIXParser(self.receiver, header_fields=[8, 9]) text = to_fix('8=FIX.4.2', '9=23', '35=A', '919=this', '955=that', '10=013') for c in text: parser.on_data_received(c) self.assertFalse(parser.is_parsing) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(6, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '23'), (35, 'A'), (919, 'this'), (955, 'that'), (10, '013')]))
def test_simple_message(self): """ Basic function test. """ parser = FIXParser(self.receiver, header_fields=[8, 9]) self.assertFalse(parser.is_parsing) # message taken from wikipedia article parser.on_data_received(to_fix('8=FIX.4.2', '9=65', '35=A', '49=SERVER', '56=CLIENT', '34=177', '52=20090107-18:15:16', '98=0', '108=30', '10=062')) self.assertFalse(parser.is_parsing) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(10, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '65'), (35, 'A'), (10, '062')]))
def test_multiple_message(self): """ Receive two messages in one data buffer """ parser = FIXParser(self.receiver, header_fields=[8, 9]) self.assertFalse(parser.is_parsing) parser.on_data_received(to_fix('8=FIX.4.2', '9=5', '35=A', '10=178', '8=FIX.4.2', '9=17', '35=E', '99=forsooth', '10=013')) self.assertFalse(parser.is_parsing) self.assertEquals(2, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(5, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '17'), (35, 'E'), (99, 'forsooth'), (10, '013')]))
def test_message_too_large(self): """ Test for too long message """ parser = FIXParser(self.receiver, header_fields=[8, 9], max_length=100) with self.assertRaises(FIXLengthTooLongError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '42=A' + 'BB'*100))
def __init__(self, name, transport, **kwargs): """ FIXProtocol initialization Args: name: A descriptive name given to this connection. transport: The transport used to interface with the networking layer. config: A dict() for the endpoint configuration. See the ROLES section in sample_config.py for examples. link_config: A dict for the link configuration. See the CONNECTIONS section in sample_config.py for examples and documentation. debug: Set this to True for debug logging. (Default: False) Raises: ValueError: name and queue are required parameters. """ if name is None: raise ValueError("name is None") if transport is None: raise ValueError("name is None") self.name = name self.transport = transport self.config = kwargs.get('config', dict()) self.link_config = kwargs.get('link_config', dict()) self._debug = kwargs.get('debug') if len(self.link_config.get('protocol_version', '')) == 0: raise ValueError('link_config missing protocol_version') # heartbeat processing self.heartbeat = self.link_config.get('heartbeat', 0) self.filter_heartbeat = False self._testrequest_id = None self._testrequest_time = None # protocol state information self._send_seqno = self.link_config.get('send_seqno', 0) self._last_send_time = datetime.datetime.now() self._received_seqno = 0 self._last_received_time = datetime.datetime.now() self._parser = FIXParser(self, header_fields=self.link_config.get( 'header_fields', None), binary_fields=self.link_config.get( 'binary_fields', None), group_fields=self.link_config.get( 'group_fields', None), max_length=self.link_config.get( 'max_length', 2048)) self._logger = logging.getLogger(__name__)
def test_partial_message(self): """ Test for partial input. """ parser = FIXParser(self.receiver, header_fields=[8, 9]) self.assertFalse(parser.is_parsing) self.assertEquals(0, self.receiver.count) parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '35=A')) self.assertEquals(0, self.receiver.count) self.assertTrue(parser.is_parsing)
def test_parser_reset(self): """ Test that the parser resets on an error """ parser = FIXParser(self.receiver, header_fields=[8, 9]) # Tag ID is not a number self.assertFalse(parser.is_parsing) with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', 'abcd=A')) self.assertFalse(parser.is_parsing)
def test_bad_binary_fields(self): """ Test bad binary fields """ parser = FIXParser(self.receiver, binary_fields=[1000], header_fields=[8, 9]) # Missing binary value portion of binary field # BUGBUG: This can cause some problems, because the parser # does not attempt to validate until the entire # field has been read in. Which this will fail because # the length goes past the end of the message. # For now, live with this. with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '1000=5', '999=11', '10=001')) # Missing binary length portion (the only time this # really impacts something is if the field has an # embedded \x01). with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '1001=1010\x011010', '10=001')) parser.on_data_received(to_fix('8=FIX.4.2', '9=14', '1001=10101010', '10=127')) self.assertIsNone(self.receiver.last_error) self.assertEquals(1, self.receiver.count) self.assertIsNotNone(self.receiver.last_received_message)
def test_message_binary_too_long(self): """ Test for message with missing binary data """ parser = FIXParser(self.receiver, header_fields=[8, 9], binary_fields=[1000], max_length=100) # length too short with self.assertRaises(FIXLengthTooLongError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '42=A', '1000=128', '1001=abababababab', '10=000'))
def test_grouped_binary_fields(self): """ Test binary fields that are in a group. """ parser = FIXParser(self.receiver, debug=True, group_fields={200: [201, 202, 99, 100]}, binary_fields=[99], header_fields=[8, 9]) text = to_fix('8=FIX.4.2', '9=80', '35=A', '200=2', '201=aabc', '99=5', '100=abcde', '201=zzzaa', '202=myname', '99=5', '100=zztop', '955=that', '10=201') parser.on_data_received(text) self.assertFalse(parser.is_parsing) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(6, len(message)) self.assertEquals(2, len(message[200])) self.assertTrue(200 in message) self.assertTrue(955 in message) subgroup = message[200][0] self.assertEquals(3, len(subgroup)) self.assertTrue(201 in subgroup) self.assertTrue(99 in subgroup) self.assertTrue(100 in subgroup) subgroup = message[200][1] self.assertEquals(4, len(subgroup)) self.assertTrue(201 in subgroup) self.assertTrue(202 in subgroup) self.assertTrue(99 in subgroup) self.assertTrue(100 in subgroup)
def test_parse_field_bad_input(self): """ Test bad _parse_field inputs """ parser = FIXParser(self.receiver) # missing '=' with self.assertRaises(FIXParserError): parser._parse_field('abcde') # bad tag id with self.assertRaises(FIXParserError): parser._parse_field('a=a') # missing tag id with self.assertRaises(FIXParserError): parser._parse_field('=a') # bad tag id with self.assertRaises(FIXParserError): parser._parse_field('10b=a')
def test_header_fields(self): """ Header field testing. """ parser = FIXParser(self.receiver, header_fields=[8, 9, 320]) parser.on_data_received(to_fix('8=FIX.4.2', '9=5', '35=A', '10=178')) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) message[320] = 'hello there' self.assertEquals(5, len(message)) # verify the order of the message items = [(k, v) for k, v in message.items()] self.assertEquals(8, items[0][0]) self.assertEquals(9, items[1][0]) self.assertEquals(320, items[2][0])
def test_partial_binary_data(self): """ Receive a piece of binary data split into two parts """ parser = FIXParser(self.receiver, binary_fields=[99], header_fields=[8, 9]) text = to_fix('8=FIX.4.2', '9=38', '35=A') + '99=5\x01100=12' text2 = to_fix('345', '919=this', '955=that', '10=198') parser.on_data_received(text) self.assertTrue(parser.is_parsing) parser.on_data_received(text2) self.assertFalse(parser.is_parsing) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(8, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '38'), (35, 'A'), (99, '5'), (100, '12345'), (919, 'this'), (955, 'that'), (10, '198')]))
def test_message_bad_binary_length(self): """ Test for message with missing binary data """ parser = FIXParser(self.receiver, header_fields=[8, 9], binary_fields=[1000], max_length=100) # length too short with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '42=A', '1000=2', '1001=abababababab', '10=000')) # length too long # This will not raise an error parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '42=A', '1000=20', '1001=ab', '10=000')) self.assertEquals(0, self.receiver.count) self.assertTrue(parser.is_parsing)
def test_multiple_groups(self): """ Test the receiving of multiple groups """ parser = FIXParser(self.receiver, group_fields={100: [101, 102, 200], 200: [201, 202], }, header_fields=[8, 9]) parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '100=2', '101=a', '102=b', '101=aa', '102=bb', '10=135')) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(4, len(message)) self.assertTrue(100 in message) self.assertEquals(2, len(message[100])) group = message[100] self.assertIsNotNone(group) self.assertEquals(2, len(group)) self.assertEquals(2, len(group[0])) self.assertTrue(101 in group[0]) self.assertTrue(102 in group[0]) self.assertEquals('a', group[0][101]) self.assertEquals('b', group[0][102]) self.assertEquals(2, len(group[1])) self.assertTrue(101 in group[1]) self.assertTrue(102 in group[1]) self.assertEquals('aa', group[1][101]) self.assertEquals('bb', group[1][102])
def test_bad_syntax(self): """ Test for various bad syntax cases """ parser = FIXParser(self.receiver, header_fields=[8, 9]) # Tag ID is not a number with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', 'abcd=A')) # Missing '=' and value portion with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4', '9=32', '35')) # Missing tag ID portion with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4', '9=32', '=A'))
def test_message_starts_incorrectly(self): """ Message must start with tag 8 """ parser = FIXParser(self.receiver, header_fields=[8, 9]) # message does not start with tag 8 with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('18=FIX.4.2', '9=32', '8=FIX.4.2', '35=A', '10=100')) # unexpected tag 8 with self.assertRaises(FIXParserError): parser.on_data_received(to_fix('8=FIX.4.2', '9=32', '8=abcdef', '35=A', '10=100'))
def test_binary_fields(self): """ Binary field testing. """ parser = FIXParser(self.receiver, binary_fields=[1000, 1010], header_fields=[8, 9]) # Test with embedded binary \x01 parser.on_data_received(to_fix('8=FIX.4.2', '9=18', '1000=5', '1001=\x01\x02\x03\x04\x05', '10=066')) self.assertEquals(1, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(5, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '18'), (1000, '5'), (1001, '\x01\x02\x03\x04\x05'), (10, '066')])) # Test with embedded '=' signs parser.on_data_received(to_fix('8=FIX.4.2', '9=18', '1000=5', '1001=31=20', '10=054')) self.assertEquals(2, self.receiver.count) message = self.receiver.last_received_message self.assertIsNotNone(message) self.assertEquals(5, len(message)) self.assertTrue(message.verify(fields=[(8, 'FIX.4.2'), (9, '18'), (1000, '5'), (1001, '31=20'), (10, '054')]))
class FIXProtocol(object): """ Implements the protocol interface. Instead of dealing directly with byte streams, we now deal with Messages. This layer is responsible for most of the administration level logic, such as handling of heartbeats/TestRequests. This class will also handle tracking of certain appliction counters, such as message sequence numbers. Attributes: name: A descriptive name given to this connection. transport: The transport interface for this connection. config: A dict() for the endpoint configuration. See the ROLES section in sample_config.py for examples. link_config: A dict for the link configuration. See the CONNECTIONS section in sample_config.py for examples and documentation. heartbeat: The heartbeat interval (in secs). If set to 0, then no heartbeat processing is performed. (Default:0) filter_heartbeat: If this is set to True, then heartbeat and TestRequest messages will not be passed on via on_message_received(). (Default: False) debug: Set this to True for debug logging. (Default: False) """ # pylint: disable=too-many-instance-attributes def __init__(self, name, transport, **kwargs): """ FIXProtocol initialization Args: name: A descriptive name given to this connection. transport: The transport used to interface with the networking layer. config: A dict() for the endpoint configuration. See the ROLES section in sample_config.py for examples. link_config: A dict for the link configuration. See the CONNECTIONS section in sample_config.py for examples and documentation. debug: Set this to True for debug logging. (Default: False) Raises: ValueError: name and queue are required parameters. """ if name is None: raise ValueError("name is None") if transport is None: raise ValueError("name is None") self.name = name self.transport = transport self.config = kwargs.get('config', dict()) self.link_config = kwargs.get('link_config', dict()) self._debug = kwargs.get('debug') if len(self.link_config.get('protocol_version', '')) == 0: raise ValueError('link_config missing protocol_version') # heartbeat processing self.heartbeat = self.link_config.get('heartbeat', 0) self.filter_heartbeat = False self._testrequest_id = None self._testrequest_time = None # protocol state information self._send_seqno = self.link_config.get('send_seqno', 0) self._last_send_time = datetime.datetime.now() self._received_seqno = 0 self._last_received_time = datetime.datetime.now() self._parser = FIXParser(self, header_fields=self.link_config.get( 'header_fields', None), binary_fields=self.link_config.get( 'binary_fields', None), group_fields=self.link_config.get( 'group_fields', None), max_length=self.link_config.get( 'max_length', 2048)) self._logger = logging.getLogger(__name__) def on_data_received(self, data): """ This is the notification from the transport. """ # pass this onto the parser self._parser.on_data_received(data) def prepare_send_message(self, message): """ Sends a message via the transport. """ self._send_seqno += 1 send_time = datetime.datetime.now() # update some of the required fields for sending message[8] = self.link_config['protocol_version'] message[34] = self._send_seqno message[52] = format_time(send_time) # apply any common fields if 'common_fields' in self.link_config: for tag, value in self.link_config['common_fields']: message[tag] = value # verify required tags for tag in self.link_config['required_fields']: # these two tags are updated when generating the binary if tag in {9, 10}: continue if tag not in message or len(str(message[tag])) == 0: raise FIXDataError(tag, 'missing field: id:{0}'.format(tag)) self._last_send_time = send_time return message def on_message_received(self, message, message_length, checksum): """ This is the callback from the parser when a message has been received. """ # verify required tags for tag in self.link_config['required_fields']: if tag not in message or len(str(message[tag])) == 0: raise FIXDataError(tag, 'missing field: id:{0}'.format(tag)) # verify the protocol version if 'protocol_version' in self.link_config: if self.link_config['protocol_version'] != message[8]: raise FIXDataError( 8, 'version mismatch: expect:{0} received:{1}'.format( self.link_config['protocol_version'], message[8] )) # verify the length and checksum if message_length != int(message[9]): raise FIXDataError( 9, 'length mismatch: expect:{0} received:{1}'.format( message_length, int(message[9]))) if checksum != int(message[10]): raise FIXDataError( 10, 'checksum mismatch: expect:{0} received:{1}'.format( checksum, message[10])) self._last_received_time = datetime.datetime.now() # Have we received our testrequest response? if (message.msg_type() == FIX.HEARTBEAT and message.get(112, '') == self._testrequest_id): self._testrequest_time = None self._testrequest_id = None # We have received a testrequest and need to send a response if message.msg_type() == FIX.TEST_REQUEST: self.transport.send_message( FIXMessage(source=[(35, FIX.HEARTBEAT), (112, message[112]), (49, self.link_config['sender_compid']), (56, self.link_config['target_compid'])])) if (not self.filter_heartbeat or (message.msg_type() not in {FIX.HEARTBEAT, FIX.TEST_REQUEST})): self.transport.on_message_received(message) def on_error_received(self, error): """ This is the callback from the parser when an error in the message has been detected. """ # pylint: disable=no-self-use raise error def on_timer_tick_received(self): """ This is the Twisted timer callback when the heartbeat interval has elapsed. """ if self.heartbeat <= 0: return now = datetime.datetime.now() # Have we received a testrequest response before we timed out? if (self._testrequest_id is not None and (now - self._testrequest_time).seconds > 2*self.heartbeat): raise FIXTimeoutError('testrequest response timeout') # if heartbeat seconds/2 have elapsed since the last time # a message was sent, send a heartbeat if (now - self._last_send_time).seconds > (self.heartbeat/2): self.transport.send_message( FIXMessage(source=[(35, FIX.HEARTBEAT), (49, self.link_config['sender_compid']), (56, self.link_config['target_compid'])])) # if heartbeat seconds + "some transmission time" have elapsed # since a message was received, send a TestRequest if (now - self._last_received_time).seconds > self.heartbeat: testrequest_id = "TR{0}".format(format_time(now)) self._testrequest_time = now self._testrequest_id = testrequest_id self.transport.send_message( FIXMessage(source=[(35, FIX.TEST_REQUEST), (112, testrequest_id), (49, self.link_config['sender_compid']), (56, self.link_config['target_compid'])]))