def index_object(self, documentId, obj, threshold=None):
"""Index an object:
- 'documentId' is the integer ID of the document
- 'obj' is the object to be indexed
- ignore threshold
"""
if self._since_field is None:
return 0
since = getattr(obj, self._since_field, None)
if safe_callable(since):
since = since()
since = self._convertDateTime(since)
until = getattr(obj, self._until_field, None)
if safe_callable(until):
until = until()
until = self._convertDateTime(until)
datum = (since, until)
old_datum = self._unindex.get(documentId, None)
if datum == old_datum: # No change? bail out!
return 0
self._increment_counter()
if old_datum is not None:
old_since, old_until = old_datum
self._removeForwardIndexEntry(old_since, old_until, documentId)
self._insertForwardIndexEntry(since, until, documentId)
self._unindex[documentId] = datum
return 1
def get_value(self, object):
try:
fields = self.index._indexed_attrs
except:
fields = [ self.index._fieldname ]
all_texts = []
for attr in fields:
text = getattr(object, attr, None)
if text is None:
continue
if safe_callable(text):
text = text()
if text is None:
continue
if text:
if isinstance(text, (list, tuple, )):
all_texts.extend(text)
else:
all_texts.append(text)
# Check that we're sending only strings
all_texts = filter(lambda text: isinstance(text, basestring), \
all_texts)
if all_texts:
return '\n'.join(all_texts)
def index_object(self, documentId, obj, threshold=None):
"""Wrapper for index_doc() handling indexing of multiple attributes.
Enter the document with the specified documentId in the index
under the terms extracted from the indexed text attributes,
each of which should yield either a string or a list of
strings (Unicode or otherwise) to be passed to index_doc().
"""
# TODO we currently ignore subtransaction threshold
# needed for backward compatibility
fields = getattr(self, '_indexed_attrs', [self._fieldname])
all_texts = []
for attr in fields:
text = getattr(obj, attr, None)
if text is None:
continue
if safe_callable(text):
text = text()
if text is not None:
if isinstance(text, (list, tuple, set)):
all_texts.extend(text)
else:
all_texts.append(text)
# Check that we're sending only strings
all_texts = [t for t in all_texts if isinstance(t, basestring)]
if all_texts:
return self.index.index_doc(documentId, all_texts)
return 0
def index_object(self, docid, obj ,threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (StringType, TupleType)):
raise TypeError('path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (ListType, TupleType)):
path = '/'+ '/'.join(path[1:])
comps = filter(None, path.split('/'))
if not self._unindex.has_key(docid):
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def index_object(self, documentId, obj, threshold=None):
try: fields = self._indexed_attrs
except: fields = [ self._fieldname ]
res = 0
for lang in self.languages:
all_texts = []
for attr in fields:
text = getattr(obj, attr, None)
if text is None:
continue
if safe_callable(text):
try: text = text(lang=lang)
except: text = text()
if text is None:
continue
if text:
if isinstance(text, (list, tuple, )):
all_texts.extend(text)
else:
all_texts.append(text)
# Check that we're sending only strings
all_texts = filter(lambda text: isinstance(text, basestring), \
all_texts)
if all_texts:
self._v_lang = lang
res += self.index.index_doc(documentId, all_texts)
self._v_lang = None
return res > 0
def get_value(self, object):
if self.index._since_field is None:
return
since = getattr(object, self.index._since_field, None)
if safe_callable(since):
since = since()
until = getattr(object, self.index._until_field, None)
if safe_callable(until):
until = until()
if not since or not until:
return
return {
'%s1' % self.index.id: since.ISO8601(),
'%s2' % self.index.id: until.ISO8601()}
def cmf_uid(self):
"""
Return the CMFUid UID of the object while making sure
it is not accidentally acquired.
"""
cmf_uid = getattr(aq_base(self.__ob), 'cmf_uid', '')
if safe_callable(cmf_uid):
return cmf_uid()
return cmf_uid
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
# PathIndex first checks for an attribute matching its id and
# falls back to getPhysicalPath only when failing to get one.
# If self.indexed_attrs is not None, it's value overrides this behavior
attrs = self.indexed_attrs
index = attrs is None and self.id or attrs[0]
path = getattr(obj, index, None)
if path is not None:
if safe_callable(path):
path = path()
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple '
'of strings: (%r, %s)' % (index, repr(path)))
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/' + '/'.join(path[1:])
comps = filter(None, path.split('/'))
# Make sure we reindex properly when path change
old_path = self._unindex.get(docid, _marker)
if old_path is not _marker:
if old_path != path:
self.unindex_object(docid, _old=old_path)
# unindex reduces length, we need to counter that
self._length.change(1)
else:
# We only get a new entry if the value wasn't there before.
# If it already existed the length is unchanged
self._length.change(1)
for i, comp in enumerate(comps):
self.insertEntry(comp, docid, i)
# Add terminator
self.insertEntry(None, docid, len(comps) - 1)
# Add full-path indexes, to optimize certain edge cases
parent_path = '/' + '/'.join(comps[:-1])
parents = self._index_parents.get(parent_path, _marker)
if parents is _marker:
self._index_parents[parent_path] = parents = IITreeSet()
parents.insert(docid)
self._index_items[path] = docid
self._unindex[docid] = path
return 1
def _get_object_paths(self, obj, attr):
try:
paths = getattr(obj, attr, ())
except AttributeError:
return _marker
if safe_callable(paths):
paths = paths()
if isinstance(paths, basestring):
return (paths,)
else:
return paths
def _get_object_datum(self, obj, attr):
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
try:
datum = getattr(obj, attr)
if safe_callable(datum):
datum = datum()
except (AttributeError, TypeError):
datum = _marker
return datum
def index_object(self, docid, obj ,threshold=100):
""" hook for (Z)Catalog """
# PathIndex first checks for an attribute matching its id and
# falls back to getPhysicalPath only when failing to get one.
# The presence of 'indexed_attrs' overrides this behavior and
# causes indexing of the custom attribute.
attrs = getattr(self, 'indexed_attrs', None)
if attrs:
index = attrs[0]
else:
index = self.id
f = getattr(obj, index, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple '
'of strings: (%r, %s)' % (index, repr(path)))
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/'+ '/'.join(path[1:])
comps = filter(None, path.split('/'))
parent_path = '/' + '/'.join(comps[:-1])
# Make sure we reindex properly when path change
if self._unindex.has_key(docid) and self._unindex.get(docid) != path:
self.unindex_object(docid)
if not self._unindex.has_key(docid):
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
# Add terminator
self.insertEntry(None, docid, len(comps)-1, parent_path, path)
self._unindex[docid] = path
return 1
def catalog_object(self, obj, uid=None, idxs=[],
update_metadata=1, pghandler=None):
mode = self.mode
if mode in (DISABLE_MODE, DUAL_MODE):
result = self.patched.catalog_object(
obj, uid, idxs, update_metadata, pghandler)
if mode == DISABLE_MODE:
return result
wrapped_object = None
if not IIndexableObject.providedBy(obj):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((obj, self.catalogtool), IIndexableObject)
if wrapper is not None:
wrapped_object = wrapper
else:
wrapped_object = obj
else:
wrapped_object = obj
conn = self.conn
catalog = self.catalog
if idxs == []:
idxs = catalog.indexes.keys()
index_data = {}
for index_name in idxs:
index = getIndex(catalog, index_name)
if index is not None:
value = index.get_value(wrapped_object)
if value in (None, 'None'):
# yes, we'll index null data...
value = None
index_data[index_name] = value
if update_metadata:
metadata = {}
for meta_name in catalog.names:
attr = getattr(wrapped_object, meta_name, MV)
if (attr is not MV and safe_callable(attr)):
attr = attr()
metadata[meta_name] = attr
# XXX Also, always index path so we can use it with the brain
# to make urls
metadata['_path'] = wrapped_object.getPhysicalPath()
index_data['_metadata'] = dumps(metadata)
uid = getUID(obj)
try:
doc = conn.get(self.catalogsid, self.catalogtype, uid)
self.registerInTransaction(uid, td.Actions.modify, doc)
except NotFoundException:
self.registerInTransaction(uid, td.Actions.add)
conn.index(index_data, self.catalogsid, self.catalogtype, uid)
if self.registry.auto_flush:
conn.refresh()
def index_object( self, documentId, obj, threshold=None ):
if self._since_field is None:
return 0
since_attr = getattr(obj, self._since_field, None)
until_attr = getattr(obj, self._until_field, None)
if not since_attr and not until_attr:
return 0
for lang in self.languages:
self._v_lang = lang
if safe_callable(since_attr):
since = since_attr()
try: since = since_attr(lang=lang)
except: pass
else:
since = since_attr
since = self._convertDateTime( since )
if safe_callable( until_attr ):
until = until_attr()
try: until = until_attr(lang=lang)
except: pass
else:
until = until_attr
until = self._convertDateTime( until )
datum = ( since, until )
old_datum = self._unindex.get( documentId, None )
if datum == old_datum: # No change? bail out!
return 0
if old_datum is not None:
old_since, old_until = old_datum
self._removeForwardIndexEntry( old_since, old_until, documentId )
self._insertForwardIndexEntry( since, until, documentId )
self._unindex[ documentId ] = datum
self._v_lang = None
return 1
def _get_generic_searchable_text(self, obj):
# Get searchable text from any object.
catalog = getToolByName(self, 'portal_catalog')
wrapper = queryMultiAdapter((obj, catalog), IIndexableObject)
if wrapper is None:
return
text = getattr(wrapper, 'SearchableText', None)
if text is None:
return
if safe_callable(text):
text = text()
return text
def index_object(self, documentId, obj, threshold=None):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
"""
returnStatus = 0
date_attr = getattr( obj, self.id, None )
if not date_attr:
return returnStatus
for lang in self.languages:
self._v_lang = lang
try:
if safe_callable( date_attr ):
date = date_attr()
try: date = date(lang=lang)
except: pass
else:
date = date_attr
ConvertedDate = self._convert( value=date, default=_marker )
except AttributeError:
ConvertedDate = _marker
oldConvertedDate = self._unindex.get( documentId, _marker )
if ConvertedDate != oldConvertedDate:
if oldConvertedDate is not _marker:
self.removeForwardIndexEntry(oldConvertedDate, documentId)
if ConvertedDate is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: ConvertedDate was there,"
" now it's not, for document with id %s" %
documentId)
if ConvertedDate is not _marker:
self.insertForwardIndexEntry( ConvertedDate, documentId )
self._unindex[documentId] = ConvertedDate
returnStatus = 1
self._v_lang = None
return returnStatus
def _get_object_keywords(self, obj, attr):
newKeywords = getattr(obj, attr, ())
if safe_callable(newKeywords):
newKeywords = newKeywords()
if (isinstance(newKeywords, StringType)
or isinstance(newKeywords, UnicodeType)): #Python 2.1 compat isinstance
return (newKeywords,)
else:
unique = {}
try:
for k in newKeywords:
unique[k] = None
except TypeError:
# Not a sequence
return (newKeywords,)
else:
return unique.keys()
def addSearchableTextField(self, icc):
st = self.context.SearchableText
if safe_callable(st):
st = st()
text = self._c(st)
icc.addContent('SearchableText', text, self.language)
f = self.context.getFile()
if not f:
return
body = str(f)
if body:
mt = f.getContentType()
if mt == 'text/plain':
icc.addContent('SearchableText', self._c(body), self.language)
else:
icc.addBinary('SearchableText', body, mt, None, self.language)
def _get_object_keywords(self, obj, attr):
newKeywords = getattr(obj, attr, ())
if safe_callable(newKeywords):
try:
newKeywords = newKeywords()
except (AttributeError, TypeError):
return ()
if not newKeywords:
return ()
elif isinstance(newKeywords, basestring):
return (newKeywords, )
else:
unique = {}
try:
for k in newKeywords:
unique[k] = None
except TypeError:
# Not a sequence
return (newKeywords, )
else:
return unique.keys()
def _get_object_keywords(self, obj, attr):
newKeywords = getattr(obj, attr, ())
if safe_callable(newKeywords):
try:
newKeywords = newKeywords()
except (AttributeError, TypeError):
return ()
if not newKeywords:
return ()
elif isinstance(newKeywords, basestring):
return (newKeywords,)
else:
unique = {}
try:
for k in newKeywords:
unique[k] = None
except TypeError:
# Not a sequence
return (newKeywords,)
else:
return unique.keys()
def get_value(self, object):
attrs = self.index.indexed_attrs
index = attrs is None and self.index.id or attrs[0]
path = getattr(object, index, None)
if path is not None:
if safe_callable(path):
path = path()
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple '
'of strings: (%r, %s)' % (index, repr(path)))
else:
try:
path = object.getPhysicalPath()
except AttributeError:
return
return {
'path': '/'.join(path),
'depth': len(path) - 1
}
def get_value(self, object):
attrs = self.index.indexed_attrs
index = attrs is None and self.index.id or attrs[0]
path = getattr(object, index, None)
if path is not None:
if safe_callable(path):
path = path()
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple '
'of strings: (%r, %s)' % (index, repr(path)))
else:
try:
path = object.getPhysicalPath()
except AttributeError:
return
return {
'path': '/'.join(path),
'depth': len(path) - 1
}
def mpi_index_object(self, docId, obj, threshold=None):
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if paths:
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
vals.append((self.id, paths, None))
return 1
return 0
def mpi_index_object(self, docId, obj, threshold=None):
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if paths:
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
vals.append((self.id, paths, None))
return 1
return 0
def index_object(self, docid, obj, threshold=100):
""" See IPluggableIndex.
"""
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (str, tuple)):
raise TypeError(
'path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/' + '/'.join(path[1:])
comps = filter(None, path.split('/'))
old_value = self._unindex.get(docid, None)
if old_value == path:
return 0
if old_value is None:
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
if isinstance(self._unindex[docid], set):
self._unindex[docid] = OOSet(self._unindex[docid])
unin = set(self._unindex[docid])
paths_set = {'/'.join(x) for x in paths}
for oldpath in unin - paths_set:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def index_object(self, docid, obj, threshold=100):
""" See IPluggableIndex.
"""
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (str, tuple)):
raise TypeError(
'path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/' + '/'.join(path[1:])
comps = filter(None, path.split('/'))
old_value = self._unindex.get(docid, None)
if old_value == path:
return 0
if old_value is None:
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
if isinstance(self._unindex[docid], set):
self._unindex[docid] = OOSet(self._unindex[docid])
unin = set(self._unindex[docid])
paths_set = {'/'.join(x) for x in paths}
for oldpath in unin - paths_set:
self.unindex_paths(docid, (oldpath,))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
unin = self._unindex[docid]
# Migrate old versions of the index to use OOSet
if isinstance(unin, set):
unin = self._unindex[docid] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in paths:
self.unindex_paths(docid, (oldpath,))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
unin = self._unindex[docid]
# Migrate old versions of the index to use OOSet
if isinstance(unin, set):
unin = self._unindex[docid] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in paths:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def __call__(self):
wrapped = IndexableObjectWrapper(self.context, self.catalog)
text = getattr(wrapped, 'SearchableText')
if safe_callable(text):
text = text()
# Archetypes object: remove id and title
if IBaseObject.providedBy(self.context):
for fieldname in ['id', 'title']:
field = self.context.Schema().getField(fieldname)
if field is None:
continue
method = field.getIndexAccessor(self.context)
value = method()
text = text.replace(value, '', 1)
# other content (e.g. dexterity): remove title
elif IContentish.providedBy(self.context):
text = text.replace(self.context.Title(), '', 1)
# Strip html tags
text = re.sub('<[^<]+?>', '', text)
return text
def addSearchableTextField(self, icc):
st = self.context.SearchableText
if safe_callable(st):
st = st()
text = self._c(st)
icc.addContent('SearchableText', text, self.language)
def index_object(self, documentId, obj, threshold=None):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
o Repeat by recurdef - a RFC2445 reccurence definition string
"""
returnStatus = 0
try:
date_attr = getattr(obj, self.id)
if safe_callable(date_attr):
date_attr = date_attr()
except AttributeError:
return returnStatus
recurdef = getattr(obj, self.attr_recurdef, None)
if safe_callable(recurdef):
recurdef = recurdef()
if not recurdef:
dates = [pydt(date_attr)]
else:
until = getattr(obj, self.attr_until, None)
if safe_callable(until):
until = until()
dates = recurrence_sequence_ical(date_attr,
recrule=recurdef,
until=until)
newvalues = IISet(map(dt2int, dates))
oldvalues = self._unindex.get(documentId, _marker)
if oldvalues is not _marker:
oldvalues = IISet(oldvalues)
if oldvalues is not _marker and newvalues is not _marker\
and not difference(newvalues, oldvalues)\
and not difference(oldvalues, newvalues):
# difference is calculated relative to first argument, so we have to
# use it twice here
return returnStatus
if oldvalues is not _marker:
for oldvalue in oldvalues:
self.removeForwardIndexEntry(oldvalue, documentId)
if newvalues is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: oldvalues was there,"
" now it's not, for document with id %s" %
documentId)
if newvalues is not _marker:
inserted = False
for value in newvalues:
self.insertForwardIndexEntry(value, documentId)
inserted = True
if inserted:
# store tuple values in reverse index entries for sorting
self._unindex[documentId] = tuple(newvalues)
returnStatus = 1
return returnStatus
def index_object(self, documentId, obj, threshold=None):
""" Index an object:
'documentId' is the integer id of the document
'obj' is the object to be indexed
'threshold' is the number of words to process between
commiting subtransactions. If 'None' subtransactions are
disabled. """
# sniff the object for our 'id', the 'document source' of the
# index is this attribute. If it smells callable, call it.
try:
source = getattr(obj, self.id)
if safe_callable(source):
source = source()
if not isinstance(source, UnicodeType):
source = str(source)
except (AttributeError, TypeError):
return 0
# sniff the object for 'id'+'_encoding'
try:
encoding = getattr(obj, self.id+'_encoding')
if safe_callable(encoding ):
encoding = str(encoding())
else:
encoding = str(encoding)
except (AttributeError, TypeError):
encoding = 'latin1'
lexicon = self.getLexicon()
splitter = lexicon.Splitter
wordScores = OIBTree()
last = None
# Run through the words and score them
for word in list(splitter(source,encoding=encoding)):
if word[0] == '\"':
last = self._subindex(word[1:-1], wordScores, last, splitter)
else:
if word==last: continue
last=word
wordScores[word]=wordScores.get(word,0)+1
# Convert scores to use wids:
widScores=IIBucket()
getWid=lexicon.getWordId
for word, score in wordScores.items():
widScores[getWid(word)]=score
del wordScores
currentWids=IISet(self._unindex.get(documentId, []))
# Get rid of document words that are no longer indexed
self.unindex_objectWids(documentId, difference(currentWids, widScores))
# Now index the words. Note that the new xIBTrees are clever
# enough to do nothing when there isn't a change. Woo hoo.
insert=self.insertForwardIndexEntry
for wid, score in widScores.items():
insert(wid, documentId, score)
# Save the unindexing info if it's changed:
wids=widScores.keys()
if wids != currentWids.keys():
self._unindex[documentId]=wids
return len(wids)
def addSearchableTextField(self, icc):
st = self.context.SearchableText
if safe_callable(st):
st = st()
text = self._c(st)
icc.addContent('SearchableText', text, self.language)
def index_object( self, documentId, obj, threshold=None ):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
o Repeat by recurdef - a RFC2445 reccurence definition string
"""
returnStatus = 0
try:
date_attr = getattr( obj, self.id )
if safe_callable( date_attr ):
date_attr = date_attr()
except AttributeError:
return returnStatus
recurdef = getattr(obj, self.attr_recurdef, None)
if safe_callable(recurdef):
recurdef = recurdef()
if not recurdef:
dates = [pydt(date_attr)]
else:
until = getattr(obj, self.attr_until, None)
if safe_callable(until):
until = until()
dates = recurrence_sequence_ical(date_attr, recrule=recurdef, until=until)
newvalues = IISet(map(dt2int, dates))
oldvalues = self._unindex.get( documentId, _marker )
if oldvalues is not _marker:
oldvalues = IISet(oldvalues)
if oldvalues is not _marker and newvalues is not _marker\
and not difference(newvalues, oldvalues)\
and not difference(oldvalues, newvalues):
# difference is calculated relative to first argument, so we have to
# use it twice here
return returnStatus
if oldvalues is not _marker:
for oldvalue in oldvalues:
self.removeForwardIndexEntry(oldvalue, documentId)
if newvalues is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: oldvalues was there,"
" now it's not, for document with id %s" %
documentId)
if newvalues is not _marker:
inserted = False
for value in newvalues:
self.insertForwardIndexEntry( value, documentId )
inserted = True
if inserted:
# store tuple values in reverse index entries for sorting
self._unindex[documentId] = tuple(newvalues)
returnStatus = 1
return returnStatus
def index_object(self, documentId, obj, threshold=None):
""" Index an object:
'documentId' is the integer id of the document
'obj' is the object to be indexed
'threshold' is the number of words to process between
commiting subtransactions. If 'None' subtransactions are
disabled. """
# sniff the object for our 'id', the 'document source' of the
# index is this attribute. If it smells callable, call it.
try:
source = getattr(obj, self.id)
if safe_callable(source):
source = source()
if not isinstance(source, UnicodeType):
source = str(source)
except (AttributeError, TypeError):
return 0
# sniff the object for 'id'+'_encoding'
try:
encoding = getattr(obj, self.id + '_encoding')
if safe_callable(encoding):
encoding = str(encoding())
else:
encoding = str(encoding)
except (AttributeError, TypeError):
encoding = 'latin1'
lexicon = self.getLexicon()
splitter = lexicon.Splitter
wordScores = OIBTree()
last = None
# Run through the words and score them
for word in list(splitter(source, encoding=encoding)):
if word[0] == '\"':
last = self._subindex(word[1:-1], wordScores, last, splitter)
else:
if word == last: continue
last = word
wordScores[word] = wordScores.get(word, 0) + 1
# Convert scores to use wids:
widScores = IIBucket()
getWid = lexicon.getWordId
for word, score in wordScores.items():
widScores[getWid(word)] = score
del wordScores
currentWids = IISet(self._unindex.get(documentId, []))
# Get rid of document words that are no longer indexed
self.unindex_objectWids(documentId, difference(currentWids, widScores))
# Now index the words. Note that the new xIBTrees are clever
# enough to do nothing when there isn't a change. Woo hoo.
insert = self.insertForwardIndexEntry
for wid, score in widScores.items():
insert(wid, documentId, score)
# Save the unindexing info if it's changed:
wids = widScores.keys()
if wids != currentWids.keys():
self._unindex[documentId] = wids
return len(wids)
