def test_detect_joined_strings_first_with_rus_letter(self): data = b"#1074#1086#1076#1086 + \r\n#1087#1072#1076)" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "водопад") token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_joined_strings_first_with_quote(self): data = b"'aaaa' + \r\n'bbbb')" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "aaaabbbb") token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_single_string_tailed_with_rus_letter(self): data = b"#1101#1101#1101)" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "эээ") token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_single_string_tailed_with_quote(self): data = b"#1101#1101#1101'...')" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "эээ...") token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_end_of_file_token(self): data = b"" t = Tokenizer(data) token = t.get_next_token() self.assertTrue(t.reader.eof) self.assertTrue(t.done) self.assertEqual(token.id, "END_FILE")
def test_detect_splitted_string_with_russian_letters(self): data = b"#1063#1077#1084 #1073#1086#1083#1100#1096#1077 #1089#1080#1083#1072, #1090 +\r\n\ #1077#1084 #1073#1086#1083#1100#1096#1077 #1086#1090#1074#1077#1090#1089#1090#1074#1077#1085#1085#1086#1089#1090#1100'.'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "Чем больше сила, тем больше ответственность.")
def test_detect_joined_strings_followed_with_single_string_quote(self): data = b"'string one'+\r\n' continues here'\r\n#1082#1086#1085#1077#1094)" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "string one continues here") token = t.get_next_token() self.assertEqual(token.value, "конец") token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_joined_strings_followed_with_single_string_rus(self): data = b"#1074#1086#1076#1086 + \r\n#1087#1072#1076\r\n#1090#1072#1088#1077#1083#1082#1072)" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "водопад") token = t.get_next_token() self.assertEqual(token.value, "тарелка") token = t.get_next_token() self.assertEqual(token.id, ")")
def check_sequence(self, fixture, sequence): """ Извлекает все токены из sequence, а затем сверяет их с fixture. """ t = Tokenizer(sequence) tokens = [] while t.has_tokens(): token = t.get_next_token() if token.value == "": tokens.append(str(token)) else: tokens.append(token.value) self.assertEqual(fixture, tokens)
def test_decode_string_with_leading_tabs(self): data = b"#9#9'from student where id = 1234'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "\t\tfrom student where id = 1234")
def test_detect_string_with_russian_letters(self): data = b"#1040 #1089#1084#1099#1089#1083'?'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "А смысл?")
def test_detect_boolean_token(self): data = b"False" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "BOOLEAN") self.assertEqual(token.value, False)
def test_detect_identifier_token(self): data = b" someObject: objClass" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "IDENTIFIER") self.assertEqual(token.value, "someObject")
def test_detect_object_token(self): data = b"\n object someObject: objClass" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "OBJECT")
def test_detect_assignment_token(self): data = b" = value" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "=")
def test_detect_sequence_entry_token(self): data = b",value" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, ",")
def test_detect_item_token(self): data = b" item\n" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "ITEM")
def test_detect_scalar_sequence_end_token(self): data = b" )" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, ")")
def test_detect_type_definition_token(self): data = b" : integer\n" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "TYPEDEF") self.assertEqual(token.value, "integer")
def test_fetch_word(self): data = b" wordToFetch123<" t = Tokenizer(data) t.move_to_next_token() word = t.fetch_word() self.assertEqual(word, b"wordToFetch123")
def test_detect_number_token(self): data = b" -123.8" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "NUMBER") self.assertEqual(token.value, -123.8)
def test_detect_identifier_sequence_end_token(self): data = b" ]" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "]")
def test_detect_string_token(self): data = b"@1SomeThing" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "STRING") self.assertEqual(token.value, "@1SomeThing")
def test_detect_item_sequence_end_token(self): data = b" >" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, ">")
def test_detect_quoted_string_token(self): data = b"'here goes quoted string'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "here goes quoted string")
def test_detect_binary_sequence_end_token(self): data = b" }" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "}")
def test_decode_russian_letters_long_code(self): data = b"#1099#1099'21'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "ыы21")
def test_detect_block_end_token(self): data = b"\n end" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.id, "END_BLOCK")
def test_decode_mixed_russian_letters(self): data = b"'Abibas - '#1101#1090#1086' '#1089#1080#1083#1072'!'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "Abibas - это сила!")
def test_decode_many_sharps(self): data = b"####1101'###'" t = Tokenizer(data) token = t.get_next_token() self.assertEqual(token.value, "###э###")