def test_tokenizer_errors(content, expected): tokenizer = ReplacementTokenizer() tokens = tokenizer.tokenize(content) assert expected == tokens