Exemple #1
0
 def test_empty(self):
     result = word_tokenize('')
     self.assertEqual(list(result), [])
Exemple #2
0
 def test_hyphens(self):
     result = word_tokenize('hi-lite')
     self.assertEqual(list(result), ['hilite'])
Exemple #3
0
 def test_conjunctions(self):
     result = word_tokenize("we're and haven't")
     self.assertEqual(list(result), ['were', 'and', 'havent'])
Exemple #4
0
    def test_only_punctuation(self):
        result = word_tokenize('.')
        self.assertEqual(list(result), [])

        result = word_tokenize('. ,\t&')
        self.assertEqual(list(result), [])
Exemple #5
0
 def test_task_example(self):
     result = word_tokenize('So call a big meeting,')
     self.assertEqual(list(result), ['so', 'call', 'a', 'big', 'meeting'])
Exemple #6
0
 def test_empty(self):
     result = word_tokenize('')
     self.assertEqual(list(result), [])
Exemple #7
0
 def test_single_word(self):
     result = word_tokenize('test')
     self.assertEqual(list(result), ['test'])
Exemple #8
0
 def test_conjunctions(self):
     result = word_tokenize("we're and haven't")
     self.assertEqual(list(result), ['were', 'and', 'havent'])
Exemple #9
0
 def test_hyphens(self):
     result = word_tokenize('hi-lite')
     self.assertEqual(list(result), ['hilite'])
Exemple #10
0
 def test_task_example(self):
     result = word_tokenize('So call a big meeting,')
     self.assertEqual(list(result), ['so', 'call', 'a', 'big', 'meeting'])
Exemple #11
0
    def test_only_punctuation(self):
        result = word_tokenize('.')
        self.assertEqual(list(result), [])

        result = word_tokenize('. ,\t&')
        self.assertEqual(list(result), [])
Exemple #12
0
 def test_single_word(self):
     result = word_tokenize('test')
     self.assertEqual(list(result), ['test'])