def tokens(self):
"""List of the tokens in the quote's :attr:`string`.
Raises
------
ValueError
If the quote's :attr:`string` is `None`.
"""
if self.string is None:
raise ValueError('No string defined on this quote yet, '
"tokens doesn't make sense.")
from brainscopypaste import tagger
return tagger.tokens(self.string)
def test_tokens():
assert tokens(sentence) == ('do', "n't", '!', 'i', 'would', "n't", '.',
'i', "'ve", 'been', 'there', '.', 'the', 'cat',
'jumped', 'over', 'the', 'fox', 'and', 'ate',
'the', 'rat')