Exemple #1
0
    def tokens(self):
        """List of the tokens in the quote's :attr:`string`.

        Raises
        ------
        ValueError
            If the quote's :attr:`string` is `None`.

        """

        if self.string is None:
            raise ValueError('No string defined on this quote yet, '
                             "tokens doesn't make sense.")
        from brainscopypaste import tagger
        return tagger.tokens(self.string)
def test_tokens():
    assert tokens(sentence) == ('do', "n't", '!', 'i', 'would', "n't", '.',
                                'i', "'ve", 'been', 'there', '.', 'the', 'cat',
                                'jumped', 'over', 'the', 'fox', 'and', 'ate',
                                'the', 'rat')