Python TokenizeOnWhitespacePunctuation.getBigrams示例

示例#1

文件： TestTokenizeOnWhitespacePunctuation.py 项目： kyajmiller/Cerebro

    def test_BigramsApplyStopwords(self):
        # set up
        teststring = 'I like cats and birds.'
        bigrams = ['cats birds']

        # test
        testtokenize = TokenizeOnWhitespacePunctuation(teststring, applyStopwords=True)
        self.assertEqual(bigrams, testtokenize.getBigrams())

示例#2

    def test_TokenizeOnWhitespacePunctuationBigrams(self):
        # set up
        teststring = 'I like cats and birds.'
        bigrams = ['i like', 'like cats', 'cats and', 'and birds']

        # test
        testtokenize = TokenizeOnWhitespacePunctuation(teststring)
        self.assertEqual(bigrams, testtokenize.getBigrams())

示例#3

文件： TestTokenizeOnWhitespacePunctuation.py 项目： kyajmiller/Cerebro

    def test_TokenizeOnWhitespacePunctuationBigrams(self):
        # set up
        teststring = 'I like cats and birds.'
        bigrams = ['i like', 'like cats', 'cats and', 'and birds']

        # test
        testtokenize = TokenizeOnWhitespacePunctuation(teststring)
        self.assertEqual(bigrams, testtokenize.getBigrams())

示例#4

    def test_BigramsApplyStopwords(self):
        # set up
        teststring = 'I like cats and birds.'
        bigrams = ['cats birds']

        # test
        testtokenize = TokenizeOnWhitespacePunctuation(teststring,
                                                       applyStopwords=True)
        self.assertEqual(bigrams, testtokenize.getBigrams())