class Lexicon(_Lexicon): family = trees.family32 # In comparison with standard Lexicon, use bigger buckets def __init__(self, *pipeline): self._wids = self.family.OI.BTree() self._words = self.family.IO.BTree() self.wordCount = Length() self._pipeline = pipeline def sourceToWordIds(self, text): if text is None: text = '' last = _text2list(text) for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): # Make sure wordCount is overridden with a BTrees.Length.Length self.wordCount = Length(self.wordCount()) # Strategically unload the length value so that we get the most # recent value written to the database to minimize conflicting wids # Because length is independent, this will load the most # recent value stored, regardless of whether MVCC is enabled self.wordCount._p_deactivate() parallel_traversal(self._wids, last) return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon): # pragma: no cover def sourceToWordIds(self, last): if last is None: last = [] if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon) : # pragma: no cover def sourceToWordIds(self, last): if last is None: last = [] if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon): # pragma : no cover def __init__(self, stop_words): super(CanopyLexicon, self).__init__() self._pipeline = [CustomStopWordRemover(stop_words)] def sourceToWordIds(self, last): if last is None: last = [] for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon) : # pragma : no cover def __init__(self, stop_words) : super(CanopyLexicon, self).__init__() self._pipeline = [CustomStopWordRemover(stop_words)] def sourceToWordIds(self, last): if last is None: last = [] for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon) : # pragma : no cover def __init__(self, stop_words) : super(CanopyLexicon, self).__init__() self._pipeline = [Splitter(), CustomStopWordRemover(stop_words), OperatorEscaper()] def sourceToWordIds(self, doc): if doc is None: doc = '' last = stringify(doc) # this is changed line for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class CanopyLexicon(Lexicon): # pragma : no cover def __init__(self, stop_words): super(CanopyLexicon, self).__init__() self._pipeline = [ Splitter(), CustomStopWordRemover(stop_words), OperatorEscaper() ] def sourceToWordIds(self, doc): if doc is None: doc = '' last = stringify(doc) # this is changed line for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): self.wordCount = Length(self.wordCount()) self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last))
class Lexicon(Persistent): """ Implementation of :class:`zope.index.text.interfaces.ILexicon`. """ def __init__(self, *pipeline): self._wids = OIBTree() # word -> wid self._words = IOBTree() # wid -> word # wid 0 is reserved for words that aren't in the lexicon (OOV -- out # of vocabulary). This can happen, e.g., if a query contains a word # we never saw before, and that isn't a known stopword (or otherwise # filtered out). Returning a special wid value for OOV words is a # way to let clients know when an OOV word appears. self.wordCount = Length() self._pipeline = pipeline def wordCount(self): """Return the number of unique terms in the lexicon.""" # overridden per instance return len(self._wids) def words(self): return self._wids.keys() def wids(self): return self._words.keys() def items(self): return self._wids.items() def sourceToWordIds(self, text): if text is None: text = '' last = _text2list(text) for element in self._pipeline: last = element.process(last) if not isinstance(self.wordCount, Length): # Make sure wordCount is overridden with a BTrees.Length.Length self.wordCount = Length(self.wordCount()) # Strategically unload the length value so that we get the most # recent value written to the database to minimize conflicting wids # Because length is independent, this will load the most # recent value stored, regardless of whether MVCC is enabled self.wordCount._p_deactivate() return list(map(self._getWordIdCreate, last)) def termToWordIds(self, text): last = _text2list(text) for element in self._pipeline: last = element.process(last) wids = [] for word in last: wids.append(self._wids.get(word, 0)) return wids def parseTerms(self, text): last = _text2list(text) for element in self._pipeline: process = getattr(element, "processGlob", element.process) last = process(last) return last def isGlob(self, word): return "*" in word or "?" in word def get_word(self, wid): return self._words[wid] def get_wid(self, word): return self._wids.get(word, 0) def globToWordIds(self, pattern): # Implement * and ? just as in the shell, except the pattern # must not start with either of these prefix = "" while pattern and pattern[0] not in "*?": prefix += pattern[0] pattern = pattern[1:] if not pattern: # There were no globbing characters in the pattern wid = self._wids.get(prefix, 0) if wid: return [wid] else: return [] if not prefix: # The pattern starts with a globbing character. # This is too efficient, so we raise an exception. raise QueryError( "pattern %r shouldn't start with glob character" % pattern) pat = prefix for c in pattern: if c == "*": pat += ".*" elif c == "?": pat += "." else: pat += re.escape(c) pat += "$" prog = re.compile(pat) keys = self._wids.keys(prefix) # Keys starting at prefix wids = [] for key in keys: if not key.startswith(prefix): break if prog.match(key): wids.append(self._wids[key]) return wids def _getWordIdCreate(self, word): wid = self._wids.get(word) if wid is None: wid = self._new_wid() self._wids[word] = wid self._words[wid] = word return wid def _new_wid(self): count = self.wordCount count.change(1) while count() in self._words: # just to be safe count.change(1) return count()