Python PythonTokenFilter Examples

Example #1

 def __init__(self, in_stream):
     PythonTokenFilter.__init__(self, in_stream)
     term = self.term = self.addAttribute(TermAttribute.class_)
     # Get tokens.
     tokens = []
     while in_stream.incrementToken():
         tokens.append(term.term())
     # Filter tokens.
     self.tokens = self.filter(tokens)
     # Setup iterator.
     self.iter = iter(self.tokens)

Example #2

File: filters.py Project: ChristopherLucas/txtorg

 def __init__(self, in_stream):
     PythonTokenFilter.__init__(self, in_stream)
     term = self.term = self.addAttribute(TermAttribute.class_)
     # Get tokens.
     tokens = []
     while in_stream.incrementToken():
         tokens.append(term.term())
     # Filter tokens.
     self.tokens = self.filter(tokens)
     # Setup iterator.
     self.iter = iter(self.tokens)

Example #3

 def __init__(self, input):
     PythonTokenFilter.__init__(self, input)
     self.input = input

Example #4

File: indexers.py Project: coady/lupyne

 def __init__(self, input):
     PythonTokenFilter.__init__(self, input)
     self.input = input