def __init__(self, name, value, request):
analyzer = WikiAnalyzer(request=request,
language=request.cfg.language_default)
value = ' '.join(
unicode('%s %s' % (word, stemmed)).strip()
for word, stemmed in analyzer.tokenize(value))
super(StemmedField, self).__init__(name, value)
class TestWikiAnalyzer(object):
word = u'HelpOnMoinTesting'
words = {word.lower(): u'',
u'help': u'',
u'on': u'',
u'moin': u'',
u'testing': u''}
def setup_class(self):
self.analyzer = WikiAnalyzer(request=self.request, language=self.request.cfg.language_default)
def test_tokenize(self):
words = self.words
tokens = list(self.analyzer.tokenize(self.word))
assert len(tokens) == len(words)
for token, stemmed in tokens:
assert token in words
assert words[token] == stemmed
class TestWikiAnalyzer(object):
word = u'HelpOnMoinTesting'
words = {
word.lower(): u'',
u'help': u'',
u'on': u'',
u'moin': u'',
u'testing': u''
}
def setup_class(self):
self.analyzer = WikiAnalyzer(language=app.cfg.language_default)
def test_tokenize(self):
words = self.words
tokens = list(self.analyzer.tokenize(self.word))
assert len(tokens) == len(words)
for token, stemmed in tokens:
assert token in words
assert words[token] == stemmed
def setup_class(self):
self.analyzer = WikiAnalyzer(request=self.request, language=self.request.cfg.language_default)
def __init__(self, name, value, request):
analyzer = WikiAnalyzer(request=request, language=request.cfg.language_default)
value = ' '.join(unicode('%s %s' % (word, stemmed)).strip() for word, stemmed in analyzer.tokenize(value))
super(StemmedField, self).__init__(name, value)
def setup_class(self):
self.analyzer = WikiAnalyzer(
request=self.request, language=self.request.cfg.language_default)
def setup_class(self):
self.analyzer = WikiAnalyzer(language=app.cfg.language_default)