def sent_to_flair(sent): """ Convert a tokenized sentence (list of words) to a Flair sentence object """ sentence = Sentence() for w in sent: token = Token(w) sentence.add_token(token) sentence.infer_space_after() return sentence
def test_sentence_infer_tokenization(): sentence = Sentence() sentence.add_token(Token('xyz')) sentence.add_token(Token('"')) sentence.add_token(Token('abc')) sentence.add_token(Token('"')) sentence.infer_space_after() assert ('xyz " abc "' == sentence.to_tokenized_string()) assert ('xyz "abc"' == sentence.to_plain_string()) sentence = Sentence('xyz " abc "') sentence.infer_space_after() assert ('xyz " abc "' == sentence.to_tokenized_string()) assert ('xyz "abc"' == sentence.to_plain_string())