def test_tokenize_ugly_message(self): message = "{1:X}{2:Y}{4\n:A:B\nC\n-}" result = tokenize(message) types = [t[0] for t in result] self.assertEqual(types, [ '{', 'NUMERIC', 'COLON', 'ALPHANUMERIC', '}', '{', 'NUMERIC', 'COLON', 'ALPHANUMERIC', '}', '{', 'NUMERIC', 'FIELD_SEPARATOR', 'ALPHANUMERIC', 'COLON', 'ALPHANUMERIC', 'NEWLINE', 'ALPHANUMERIC', 'TERMINAL_FIELD', '}' ])
def test_tokenize_simple_message(self): message = "{1:F01HBOSXXXXAXXX9999999999}" result = tokenize(message) types = [t[0] for t in result] self.assertEqual(types, ['{', 'NUMERIC', 'COLON', 'ALPHANUMERIC', '}'])