def createComponents(_self, fieldName):
source = WhitespaceTokenizer()
return Analyzer.TokenStreamComponents(
source,
ICUNormalizer2Filter(
source,
Normalizer2.getInstance(
None, "nfc", UNormalizationMode2.DECOMPOSE)))
def createComponents(self, fieldName):
source = StandardTokenizer()
result = EnglishPossessiveFilter(source)
result = LowerCaseFilter(result)
result = DiacriticFilter(result)
result = StopFilter(result, self.stopwords)
if self.stemExclusionSet.isEmpty() is False:
result = SetKeywordMarkerFilter(result, self.stemExclusionSet)
result = PorterStemFilter(result)
return Analyzer.TokenStreamComponents(source, result)
def createComponents(self, fieldName, reader):
class _stopFilter(PythonFilteringTokenFilter):
def __init__(_self, tokenStream):
super(_stopFilter, _self).__init__(Version.LUCENE_CURRENT,
tokenStream)
_self.termAtt = _self.addAttribute(CharTermAttribute.class_)
def accept(_self):
return _self.termAtt.toString() != "stop"
source = WhitespaceTokenizer(Version.LUCENE_CURRENT, reader)
return Analyzer.TokenStreamComponents(source, _stopFilter(source))
def createComponents(_self, fieldName):
source = WhitespaceTokenizer()
return Analyzer.TokenStreamComponents(source,
ICUFoldingFilter(source))
def createComponents(_self, fieldName, reader):
source = WhitespaceTokenizer(Version.LUCENE_CURRENT, reader)
return Analyzer.TokenStreamComponents(
source, ICUNormalizer2Filter(source))
def createComponents(_self, fieldName):
return Analyzer.TokenStreamComponents(_tokenizer())
def createComponents(self, fieldName):
source = LowerCaseTokenizer()
return Analyzer.TokenStreamComponents(source, PayloadFilter(source, fieldName))
def createComponents(self, fieldName, reader):
source = LowerCaseTokenizer(Version.LUCENE_CURRENT, reader)
return Analyzer.TokenStreamComponents(source,
PayloadFilter(source, fieldName))