def _process_token(self, token): if self.krovetz_stemming: try: result = pyndri.stem(token) except UnicodeEncodeError as e: logging.error(e) result = None else: result = token return result
def test_stemming(self): self.assertEqual(pyndri.stem('predictions'), 'prediction') self.assertEqual(pyndri.stem('marketing'), 'marketing') self.assertEqual(pyndri.stem('strategies'), 'strategy')
import pyndri print pyndri.stem('predictions') # prediction print pyndri.stem('marketing') # marketing print pyndri.stem('strategies') # strategy
def _process_token(self, token): if self.krovetz_stemming: return pyndri.stem(token) else: return token