Python TercomTokenizer примеры использования

Пример #1

def test_ter_tokenizer_default(input, expected):
    tokenizer = TercomTokenizer(no_punct=True)
    assert tokenizer(input) == expected

Пример #2

def test_ter_tokenizer_default(input, expected):
    tokenizer = TercomTokenizer(normalized=True, asian_support=True)
    assert tokenizer(input) == expected

Пример #3

def test_ter_tokenizer_norm(input, expected):
    tokenizer = TercomTokenizer(normalized=True)
    assert tokenizer(input) == expected