def test_tokenize():
tokens = tokenize("My hovercraft is full of eels.")
expected = "My hovercraft is full of eels .".split()
for obs, exp in zip(tokens, expected):
assert_equal(obs, {"token": exp})
def test_tokenize():
tokens = tokenize("My hovercraft is full of eels.")
expected = "My hovercraft is full of eels .".split()
assert_equal(tokens, expected)
def test_tokenize():
tokens = tokenize("My hovercraft is full of eels.")
expected = "My hovercraft is full of eels .".split()
for obs, exp in zip(tokens, expected):
assert_equal(obs, {"token": exp})
