class FilteredSetBase(Persistent): # A pre-calculated result list based on an expression. def __init__(self, id, expr): self.id = id self.expr = expr self.clear() def clear(self): self.ids = IITreeSet() def index_object(self, documentId, obj): raise NotImplementedError('index_object not defined') def unindex_object(self, documentId): try: self.ids.remove(documentId) except KeyError: pass def getId(self): return self.id def getExpression(self): # Get the expression. return self.expr def getIds(self): # Get the IDs of all objects for which the expression is True. return self.ids def getType(self): return self.meta_type def setExpression(self, expr): # Set the expression. self.expr = expr def __repr__(self): return '{0}: ({1}) {2}'.format( self.id, self.expr, list(map(None, self.ids)) ) __str__ = __repr__
class BooleanIndex(UnIndex): """Index for booleans self._index = set([documentId1, documentId2]) self._unindex = {documentId:[True/False]} self._length is the length of the unindex self._index_length is the length of the index False doesn't have actual entries in _index. """ meta_type = "BooleanIndex" manage_options = ( {'label': 'Settings', 'action': 'manage_main'}, {'label': 'Browse', 'action': 'manage_browse'}, ) query_options = ["query"] manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals()) manage_main._setName('manage_main') manage_browse = DTMLFile('../dtml/browseIndex', globals()) _index_value = 1 _index_length = None def clear(self): self._index = IITreeSet() self._index_length = BTrees.Length.Length() self._index_value = 1 self._unindex = IIBTree() self._length = BTrees.Length.Length() if self._counter is None: self._counter = BTrees.Length.Length() else: self._increment_counter() def histogram(self): """Return a mapping which provides a histogram of the number of elements found at each point in the index. """ histogram = {} indexed = bool(self._index_value) histogram[indexed] = self._index_length.value histogram[not indexed] = self._length.value - self._index_length.value return histogram def _invert_index(self, documentId=None): self._index_value = indexed = int(not self._index_value) self._index.clear() length = 0 for rid, value in self._unindex.iteritems(): if value == indexed: self._index.add(rid) length += 1 # documentId is the rid of the currently processed object that # triggered the invert. in the case of unindexing, the rid hasn't # been removed from the unindex yet. While indexing, the rid will # be added to the index and unindex after this method is done if documentId is not None: self._index.remove(documentId) length -= 1 self._index_length = BTrees.Length.Length(length) def insertForwardIndexEntry(self, entry, documentId): """If the value matches the indexed one, insert into treeset """ # When we get the first entry, decide to index the opposite of what # we got, as indexing zero items is fewer than one. if self._length.value == 0: self._index_value = int(not bool(entry)) # if the added entry value is index value, insert it into index if bool(entry) is bool(self._index_value): self._index_length.change(1) self._index.insert(documentId) # insert value into global unindex (before computing index invert) self._unindex[documentId] = entry self._length.change(1) # is the index (after adding the current entry) larger than 60% # of the total length? than switch the indexed value if bool(entry) is bool(self._index_value): if (self._index_length.value) >= ((self._length.value) * 0.6): self._invert_index() def removeForwardIndexEntry(self, entry, documentId, check=True): """Take the entry provided and remove any reference to documentId in its entry in the index. """ if bool(entry) is bool(self._index_value): try: self._index.remove(documentId) self._index_length.change(-1) except ConflictError: raise except Exception: LOG.exception( '%s: unindex_object could not remove documentId %s ' 'from index %s. This should not happen.' % ( self.__class__.__name__, str(documentId), str(self.id))) elif check: # is the index (after removing the current entry) larger than # 60% of the total length? than switch the indexed value if (self._index_length.value) <= ((self._length.value - 1) * 0.6): self._invert_index(documentId) return def _index_object(self, documentId, obj, threshold=None, attr=''): """ index and object 'obj' with integer id 'documentId'""" returnStatus = 0 # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj, attr) # Make it boolean, int as an optimization if datum is not _marker: datum = int(bool(datum)) # We don't want to do anything that we don't have to here, so we'll # check to see if the new and existing information is the same. oldDatum = self._unindex.get(documentId, _marker) if datum != oldDatum: if oldDatum is not _marker: self.removeForwardIndexEntry(oldDatum, documentId, check=False) if datum is _marker: try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.error('Should not happen: oldDatum was there, now ' 'its not, for document with id %s' % documentId) if datum is not _marker: self.insertForwardIndexEntry(datum, documentId) returnStatus = 1 return returnStatus def unindex_object(self, documentId): """ Unindex the object with integer id 'documentId' and don't raise an exception if we fail """ unindexRecord = self._unindex.get(documentId, _marker) if unindexRecord is _marker: return None self._increment_counter() self.removeForwardIndexEntry(unindexRecord, documentId) try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.debug('Attempt to unindex nonexistent document' ' with id %s' % documentId, exc_info=True) def query_index(self, record, resultset=None): index = self._index indexed = self._index_value for key in record.keys: if bool(key) is bool(indexed): # If we match the indexed value, check index return intersection(index, resultset) else: # Otherwise, remove from resultset or _unindex if resultset is None: return union(difference(self._unindex, index), IISet([])) else: return difference(resultset, index) return IISet() def indexSize(self): """Return distinct values, as an optimization we always claim 2.""" return 2 def items(self): # return a list of value to int set of rid tuples indexed = self._index_value items = [(bool(indexed), self._index)] false = IISet() for rid, value in self._unindex.iteritems(): if value != indexed: false.add(rid) items.append((not bool(indexed), false)) return items
class BooleanIndex(UnIndex): """Index for booleans self._index = set([documentId1, documentId2]) self._unindex = {documentId:[True/False]} self._length is the length of the unindex self._index_length is the length of the index False doesn't have actual entries in _index. """ meta_type = "BooleanIndex" manage_options = ( { 'label': 'Settings', 'action': 'manage_main' }, { 'label': 'Browse', 'action': 'manage_browse' }, ) query_options = ["query"] manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals()) manage_main._setName('manage_main') manage_browse = DTMLFile('../dtml/browseIndex', globals()) _index_value = 1 _index_length = None def clear(self): self._index = IITreeSet() self._index_length = BTrees.Length.Length() self._index_value = 1 self._unindex = IIBTree() self._length = BTrees.Length.Length() def histogram(self): """Return a mapping which provides a histogram of the number of elements found at each point in the index. """ histogram = {} indexed = bool(self._index_value) histogram[indexed] = self._index_length.value histogram[not indexed] = self._length.value - self._index_length.value return histogram def _invert_index(self, documentId=None): self._index_value = indexed = int(not self._index_value) self._index.clear() length = 0 for rid, value in self._unindex.iteritems(): if value == indexed: self._index.add(rid) length += 1 # documentId is the rid of the currently processed object that # triggered the invert. in the case of unindexing, the rid hasn't # been removed from the unindex yet. While indexing, the rid will # be added to the index and unindex after this method is done if documentId is not None: self._index.remove(documentId) length -= 1 self._index_length = BTrees.Length.Length(length) def _inline_migration(self): self._length = BTrees.Length.Length(len(self._unindex.keys())) self._index_length = BTrees.Length.Length(len(self._index)) if self._index_length.value > (self._length.value / 2): self._index_value = 1 self._invert_index() else: # set an instance variable self._index_value = 1 def insertForwardIndexEntry(self, entry, documentId): """If the value matches the indexed one, insert into treeset """ # when we get the first entry, decide to index the opposite of what # we got, as indexing zero items is fewer than one # BBB inline migration if self._index_length is None: self._inline_migration() if self._length.value == 0: self._index_value = int(not bool(entry)) # if the added entry value is index value, insert it into index if bool(entry) is bool(self._index_value): self._index_length.change(1) self._index.insert(documentId) # insert value into global unindex (before computing index invert) self._unindex[documentId] = entry self._length.change(1) # is the index (after adding the current entry) larger than 60% # of the total length? than switch the indexed value if bool(entry) is bool(self._index_value): if (self._index_length.value) >= ((self._length.value) * 0.6): self._invert_index() def removeForwardIndexEntry(self, entry, documentId, check=True): """Take the entry provided and remove any reference to documentId in its entry in the index. """ index_length = self._index_length if index_length is None: self._inline_migration() if bool(entry) is bool(self._index_value): try: self._index.remove(documentId) # BBB inline migration length = self._index_length length.change(-1) except ConflictError: raise except Exception: LOG.exception( '%s: unindex_object could not remove ' 'documentId %s from index %s. This ' 'should not happen.' % (self.__class__.__name__, str(documentId), str(self.id))) elif check: length = self._length.value index_length = self._index_length.value # is the index (after removing the current entry) larger than # 60% of the total length? than switch the indexed value if (index_length) <= ((length - 1) * 0.6): self._invert_index(documentId) return def _index_object(self, documentId, obj, threshold=None, attr=''): """ index and object 'obj' with integer id 'documentId'""" returnStatus = 0 # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj, attr) # Make it boolean, int as an optimization if datum is not _marker: datum = int(bool(datum)) # We don't want to do anything that we don't have to here, so we'll # check to see if the new and existing information is the same. oldDatum = self._unindex.get(documentId, _marker) if datum != oldDatum: if oldDatum is not _marker: self.removeForwardIndexEntry(oldDatum, documentId, check=False) if datum is _marker: try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.error('Should not happen: oldDatum was there, now ' 'its not, for document with id %s' % documentId) if datum is not _marker: self.insertForwardIndexEntry(datum, documentId) returnStatus = 1 return returnStatus def unindex_object(self, documentId): """ Unindex the object with integer id 'documentId' and don't raise an exception if we fail """ unindexRecord = self._unindex.get(documentId, _marker) if unindexRecord is _marker: return None self.removeForwardIndexEntry(unindexRecord, documentId) try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.debug('Attempt to unindex nonexistent document' ' with id %s' % documentId, exc_info=True) def _apply_index(self, request, resultset=None): record = parseIndexRequest(request, self.id, self.query_options) if record.keys is None: return None index = self._index indexed = self._index_value for key in record.keys: if bool(key) is bool(indexed): # If we match the indexed value, check index return (intersection(index, resultset), (self.id, )) else: # Otherwise, remove from resultset or _unindex if resultset is None: return (union(difference(self._unindex, index), IISet([])), (self.id, )) else: return (difference(resultset, index), (self.id, )) return (IISet(), (self.id, )) def indexSize(self): """Return distinct values, as an optimization we always claim 2.""" return 2 def items(self): # return a list of value to int set of rid tuples indexed = self._index_value items = [(bool(indexed), self._index)] false = IISet() for rid, value in self._unindex.iteritems(): if value != indexed: false.add(rid) items.append((not bool(indexed), false)) return items
class BooleanIndex(UnIndex): """Index for booleans self._index = set([documentId1, documentId2]) self._unindex = {documentId:[True/False]} self._length is the length of the unindex self._index_length is the length of the index self._index_value is the indexed value The document ids in self._index have self._index_value as their value. Since there are only two possible values (True/False), the index only stores a forward index for the less common value. It starts off with the opposite of value of the first document and later checks and inverts itself, if more than 60% of all documents now have the indexed value. It does the inversion at 60% to avoid inverting itself constantly for an index that has a roughly equal 50/50 split. """ meta_type = 'BooleanIndex' manage_options = ( { 'label': 'Settings', 'action': 'manage_main' }, { 'label': 'Browse', 'action': 'manage_browse' }, ) query_options = ['query'] manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals()) manage_main._setName('manage_main') manage_browse = DTMLFile('../dtml/browseIndex', globals()) _index_value = 1 _index_length = None def clear(self): self._index = IITreeSet() self._index_length = BTrees.Length.Length() self._index_value = 1 self._unindex = IIBTree() self._length = BTrees.Length.Length() if self._counter is None: self._counter = BTrees.Length.Length() else: self._increment_counter() def histogram(self): """Return a mapping which provides a histogram of the number of elements found at each point in the index. """ histogram = {} indexed = bool(self._index_value) histogram[indexed] = self._index_length.value histogram[not indexed] = self._length.value - self._index_length.value return histogram def _invert_index(self, documentId=None): self._index_value = indexed = int(not self._index_value) self._index.clear() length = 0 for rid, value in self._unindex.iteritems(): if value == indexed: self._index.add(rid) length += 1 # documentId is the rid of the currently processed object that # triggered the invert. in the case of unindexing, the rid hasn't # been removed from the unindex yet. While indexing, the rid will # be added to the index and unindex after this method is done if documentId is not None: self._index.remove(documentId) length -= 1 self._index_length = BTrees.Length.Length(length) def insertForwardIndexEntry(self, entry, documentId): """If the value matches the indexed one, insert into treeset """ # When we get the first entry, decide to index the opposite of what # we got, as indexing zero items is fewer than one. if self._length.value == 0: self._index_value = int(not bool(entry)) # if the added entry value is index value, insert it into index if bool(entry) is bool(self._index_value): self._index_length.change(1) self._index.insert(documentId) # insert value into global unindex (before computing index invert) self._unindex[documentId] = entry self._length.change(1) # is the index (after adding the current entry) larger than 60% # of the total length? than switch the indexed value if bool(entry) is bool(self._index_value): if (self._index_length.value) >= ((self._length.value) * 0.6): self._invert_index() def removeForwardIndexEntry(self, entry, documentId, check=True): """Take the entry provided and remove any reference to documentId in its entry in the index. """ if bool(entry) is bool(self._index_value): try: self._index.remove(documentId) self._index_length.change(-1) except ConflictError: raise except Exception: LOG.exception( '%(context)s: unindex_object could not ' 'remove documentId %(doc_id)s from ' 'index %(index)r. This should not ' 'happen.', dict(context=self.__class__.__name__, doc_id=documentId, index=self.id)) elif check: # is the index (after removing the current entry) larger than # 60% of the total length? than switch the indexed value if (self._index_length.value) <= ((self._length.value - 1) * 0.6): self._invert_index(documentId) return def _index_object(self, documentId, obj, threshold=None, attr=''): """ index and object 'obj' with integer id 'documentId'""" returnStatus = 0 # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj, attr) # Make it boolean, int as an optimization if datum is not _marker: datum = int(bool(datum)) # We don't want to do anything that we don't have to here, so we'll # check to see if the new and existing information is the same. oldDatum = self._unindex.get(documentId, _marker) if datum != oldDatum: if oldDatum is not _marker: self.removeForwardIndexEntry(oldDatum, documentId, check=False) if datum is _marker: try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.error( 'Should not happen: oldDatum was there, now ' 'its not, for document with id %s', documentId) if datum is not _marker: self.insertForwardIndexEntry(datum, documentId) returnStatus = 1 return returnStatus def unindex_object(self, documentId): """ Unindex the object with integer id 'documentId' and don't raise an exception if we fail """ unindexRecord = self._unindex.get(documentId, _marker) if unindexRecord is _marker: return None self._increment_counter() self.removeForwardIndexEntry(unindexRecord, documentId) try: del self._unindex[documentId] self._length.change(-1) except ConflictError: raise except Exception: LOG.debug('Attempt to unindex nonexistent document' ' with id %s', documentId, exc_info=True) def query_index(self, record, resultset=None): index = self._index indexed = self._index_value for key in record.keys: if bool(key) is bool(indexed): # If we match the indexed value, check index return intersection(index, resultset) else: # Otherwise, remove from resultset or _unindex if resultset is None: return union(difference(self._unindex, index), IISet([])) else: return intersection(difference(resultset, index), self._unindex) return IISet() def indexSize(self): """Return distinct values, as an optimization we always claim 2.""" return 2 def items(self): # return a list of value to int set of rid tuples indexed = self._index_value items = [(bool(indexed), self._index)] false = IISet() for rid, value in self._unindex.iteritems(): if value != indexed: false.add(rid) items.append((not bool(indexed), false)) return items def uniqueValues(self, name=None, withLengths=0): """returns the unique values for name if withLengths is true, returns a sequence of tuples of (value, length) """ if name is None: name = self.id elif name != self.id: return indexed = bool(self._index_value) unique_values = (indexed, not indexed) if not withLengths: for key in unique_values: yield key else: for key in unique_values: ilen = len(self._index) if key is indexed: yield (key, ilen) else: ulen = len(self._unindex) yield (key, ulen - ilen)
class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match (i.e., they returned None for both); - Objects which match after a given time (i.e., they returned None for the end date); - Objects which match until a given time (i.e., they returned None for the start date); - Objects which match only during a specific interval. """ implements(IDateRangeIndex) security = ClassSecurityInfo() meta_type = "DateRangeIndex" query_options = ('query', ) manage_options = ({ 'label': 'Properties', 'action': 'manage_indexProperties' }, ) since_field = until_field = None # int(DateTime('1000/1/1 0:00 GMT-12').millis() / 1000 / 60) floor_value = -510162480 # int(DateTime('2499/12/31 0:00 GMT+12').millis() / 1000 / 60) ceiling_value = 278751600 def __init__(self, id, since_field=None, until_field=None, caller=None, extra=None, floor_value=None, ceiling_value=None): if extra: since_field = extra.since_field until_field = extra.until_field floor_value = getattr(extra, 'floor_value', None) ceiling_value = getattr(extra, 'ceiling_value', None) self._setId(id) self._edit(since_field, until_field, floor_value, ceiling_value) self.clear() security.declareProtected(view, 'getSinceField') def getSinceField(self): """Get the name of the attribute indexed as start date. """ return self._since_field security.declareProtected(view, 'getUntilField') def getUntilField(self): """Get the name of the attribute indexed as end date. """ return self._until_field security.declareProtected(view, 'getFloorValue') def getFloorValue(self): """""" return self.floor_value security.declareProtected(view, 'getCeilingValue') def getCeilingValue(self): """""" return self.ceiling_value manage_indexProperties = DTMLFile('manageDateRangeIndex', _dtmldir) security.declareProtected(manage_zcatalog_indexes, 'manage_edit') def manage_edit(self, since_field, until_field, floor_value, ceiling_value, REQUEST): """ """ self._edit(since_field, until_field, floor_value, ceiling_value) REQUEST['RESPONSE'].redirect('%s/manage_main' '?manage_tabs_message=Updated' % REQUEST.get('URL2')) security.declarePrivate('_edit') def _edit(self, since_field, until_field, floor_value=None, ceiling_value=None): """Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field if floor_value is not None: self.floor_value = int(floor_value) if ceiling_value is not None: self.ceiling_value = int(ceiling_value) security.declareProtected(manage_zcatalog_indexes, 'clear') def clear(self): """ Start over fresh. """ self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length() # # PluggableIndexInterface implementation (XXX inherit assertions?) # def getEntryForObject(self, documentId, default=None): """ Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get(documentId, default) def index_object(self, documentId, obj, threshold=None): """ Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr(obj, self._since_field, None) if safe_callable(since): since = since() since = self._convertDateTime(since) until = getattr(obj, self._until_field, None) if safe_callable(until): until = until() until = self._convertDateTime(until) datum = (since, until) old_datum = self._unindex.get(documentId, None) if datum == old_datum: # No change? bail out! return 0 if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry(old_since, old_until, documentId) self._insertForwardIndexEntry(since, until, documentId) self._unindex[documentId] = datum return 1 def unindex_object(self, documentId): """ Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get(documentId, None) if datum is None: return since, until = datum self._removeForwardIndexEntry(since, until, documentId) del self._unindex[documentId] def uniqueValues(self, name=None, withLengths=0): """ Return a list of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '(value, length)'. """ if not name in (self._since_field, self._until_field): return [] if name == self._since_field: t1 = self._since t2 = self._since_only else: t1 = self._until t2 = self._until_only result = [] if not withLengths: result.extend(t1.keys()) result.extend(t2.keys()) else: for key in t1.keys(): set = t1[key] if isinstance(set, int): length = 1 else: length = len(set) result.append((key, length)) for key in t2.keys(): set = t2[key] if isinstance(set, int): length = 1 else: length = len(set) result.append((key, length)) return tuple(result) def _cache_key(self, catalog): cid = catalog.getId() counter = getattr(aq_base(catalog), 'getCounter', None) if counter is not None: return '%s_%s' % (cid, counter()) return cid def _apply_index(self, request, resultset=None): """ Apply the index to query parameters given in 'request', which should be a mapping object. If the request does not contain the needed parameters, then return None. Otherwise return two objects. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ iid = self.id record = parseIndexRequest(request, iid, self.query_options) if record.keys is None: return None term = self._convertDateTime(record.keys[0]) REQUEST = aq_get(self, 'REQUEST', None) if REQUEST is not None: catalog = aq_parent(aq_parent(aq_inner(self))) if catalog is not None: key = self._cache_key(catalog) cache = REQUEST.get(key, None) tid = isinstance(term, int) and term / 10 or 'None' if resultset is None: cachekey = '_daterangeindex_%s_%s' % (iid, tid) else: cachekey = '_daterangeindex_inverse_%s_%s' % (iid, tid) if cache is None: cache = REQUEST[key] = RequestCache() else: cached = cache.get(cachekey, None) if cached is not None: if resultset is None: return (cached, (self._since_field, self._until_field)) else: return (difference(resultset, cached), (self._since_field, self._until_field)) if resultset is None: # Aggregate sets for each bucket separately, to avoid # large-small union penalties. until_only = multiunion(self._until_only.values(term)) since_only = multiunion(self._since_only.values(None, term)) until = multiunion(self._until.values(term)) # Total result is bound by resultset if REQUEST is None: until = intersection(resultset, until) since = multiunion(self._since.values(None, term)) bounded = intersection(until, since) # Merge from smallest to largest. result = multiunion( [bounded, until_only, since_only, self._always]) if REQUEST is not None and catalog is not None: cache[cachekey] = result return (result, (self._since_field, self._until_field)) else: # Compute the inverse and subtract from res until_only = multiunion(self._until_only.values(None, term - 1)) since_only = multiunion(self._since_only.values(term + 1)) until = multiunion(self._until.values(None, term - 1)) since = multiunion(self._since.values(term + 1)) result = multiunion([until_only, since_only, until, since]) if REQUEST is not None and catalog is not None: cache[cachekey] = result return (difference(resultset, result), (self._since_field, self._until_field)) def _insert_migrate(self, tree, key, value): treeset = tree.get(key, None) if treeset is None: tree[key] = value else: if isinstance(treeset, int): tree[key] = IITreeSet((treeset, value)) elif isinstance(treeset, IISet): tree[key] = IITreeSet(treeset) tree[key].insert(value) else: treeset.insert(value) def _insertForwardIndexEntry(self, since, until, documentId): """Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert(documentId) elif since is None: self._insert_migrate(self._until_only, until, documentId) elif until is None: self._insert_migrate(self._since_only, since, documentId) else: self._insert_migrate(self._since, since, documentId) self._insert_migrate(self._until, until, documentId) def _remove_delete(self, tree, key, value): treeset = tree.get(key, None) if treeset is not None: if isinstance(treeset, int): del tree[key] else: treeset.remove(value) if not treeset: del tree[key] def _removeForwardIndexEntry(self, since, until, documentId): """Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove(documentId) elif since is None: self._remove_delete(self._until_only, until, documentId) elif until is None: self._remove_delete(self._since_only, since, documentId) else: self._remove_delete(self._since, since, documentId) self._remove_delete(self._until, until, documentId) def _convertDateTime(self, value): if value is None: return value if isinstance(value, (str, datetime)): dt_obj = DateTime(value) value = dt_obj.millis() / 1000 / 60 # flatten to minutes elif isinstance(value, DateTime): value = value.millis() / 1000 / 60 # flatten to minutes if value > MAX32 or value < -MAX32: # t_val must be integer fitting in the 32bit range raise OverflowError('%s is not within the range of dates allowed' 'by a DateRangeIndex' % value) value = int(value) # handle values outside our specified range if value > self.ceiling_value: return None elif value < self.floor_value: return None return value
class MessageStorage(Persistent, Location): interface.implements(IMessageStorage) notify = True principalId = None def __init__(self, principalId): self.index = OIBTree() self.messages = IOBTree() self.services = OOBTree() self.readstatus = IITreeSet() self.principalId = principalId self._next = Length(1) @Lazy def readstatus(self): self.readstatus = IITreeSet() return self.readstatus @property def principal(self): try: return getUtility(IAuthentication).getPrincipal(self.principalId) except: return None @property def unread(self): unread = 0 for serviceId in self.services.keys(): service = self.getService(serviceId) unread = unread + service.unread() return unread def getMessage(self, messageId): return self.messages.get(messageId) def getServiceIds(self): return list(self.services.keys()) def getService(self, serviceId): service = self.services.get(serviceId) if not IMessageService.providedBy(service): factory = getUtility(IMessageServiceFactory, serviceId) service = factory(self) self.services[serviceId] = service return service def create(self, serviceId, **data): """ create and append message to storage """ id = self._next() self._next.change(1) service = self.getService(serviceId) msg = service.create(**data) date = datetime.now(ITZInfo(self.principal, pytz.utc)) while date in self.index: date = date + timedelta msg.__id__ = id msg.__date__ = date self.index[date] = id self.messages[id] = msg self.readstatus.insert(id) service.append(msg) event.notify(MessageCreatedEvent(msg, self)) return id def remove(self, messageId): message = self.messages.get(messageId) if message is None: return else: self.clearReadStatus(message) del self.index[message.__date__] del self.messages[message.__id__] for serviceId in self.services.keys(): service = self.getService(serviceId) service.remove(message) event.notify(MessageRemovedEvent(message, self)) def readStatus(self, message): return message.__id__ in self.readstatus def clearReadStatus(self, message): if message.__id__ not in self.readstatus: return idx = message.__date__ for serviceId in self.services.keys(): service = self.getService(serviceId) if idx in service.index and service.unread() > 0: service.unread.change(-1) self.readstatus.remove(message.__id__)
class DateRangeIndex(UnIndex): """ Index a date range, such as the canonical "effective-expiration" range in the CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match ( i.e., they returned None for both ); - Objects which match after a given time ( i.e., they returned None for the end date ); - Objects which match until a given time ( i.e., they returned None for the start date ); - Objects which match only during a specific interval. """ __implements__ = ( PluggableIndex.PluggableIndexInterface, ) security = ClassSecurityInfo() meta_type = "DateRangeIndex" manage_options= ( { 'label' : 'Properties' , 'action' : 'manage_indexProperties' } , ) query_options = ['query'] since_field = until_field = None def __init__(self, id, since_field=None, until_field=None, caller=None, extra=None): if extra: since_field = extra.since_field until_field = extra.until_field self._setId(id) self._edit(since_field, until_field) self.clear() security.declareProtected( VIEW_PERMISSION , 'getSinceField' ) def getSinceField( self ): """ """ return self._since_field security.declareProtected( VIEW_PERMISSION , 'getUntilField' ) def getUntilField( self ): """ """ return self._until_field manage_indexProperties = DTMLFile( 'manageDateRangeIndex', _dtmldir ) security.declareProtected( INDEX_MGMT_PERMISSION , 'manage_edit' ) def manage_edit( self, since_field, until_field, REQUEST ): """ """ self._edit( since_field, until_field ) REQUEST[ 'RESPONSE' ].redirect( '%s/manage_main' '?manage_tabs_message=Updated' % REQUEST.get('URL2') ) security.declarePrivate( '_edit' ) def _edit( self, since_field, until_field ): """ Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field security.declareProtected( INDEX_MGMT_PERMISSION , 'clear' ) def clear( self ): """ Start over fresh. """ self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints # # PluggableIndexInterface implementation (XXX inherit assertions?) # def getEntryForObject( self, documentId, default=None ): """ Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get( documentId, default ) def index_object( self, documentId, obj, threshold=None ): """ Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr( obj, self._since_field, None ) if callable( since ): since = since() since = self._convertDateTime( since ) until = getattr( obj, self._until_field, None ) if callable( until ): until = until() until = self._convertDateTime( until ) datum = ( since, until ) old_datum = self._unindex.get( documentId, None ) if datum == old_datum: # No change? bail out! return 0 if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry( old_since, old_until, documentId ) self._insertForwardIndexEntry( since, until, documentId ) self._unindex[ documentId ] = datum return 1 def unindex_object( self, documentId ): """ Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get( documentId, None ) if datum is None: return since, until = datum self._removeForwardIndexEntry( since, until, documentId ) del self._unindex[ documentId ] def uniqueValues( self, name=None, withLengths=0 ): """ Return a list of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '( value, length )'. """ if not name in ( self._since_field, self._until_field ): return [] if name == self._since_field: t1 = self._since t2 = self._since_only else: t1 = self._until t2 = self._until_only result = [] IntType = type( 0 ) if not withValues: result.extend( t1.keys() ) result.extend( t2.keys() ) else: for key in t1.keys(): set = t1[ key ] if type( set ) is IntType: length = 1 else: length = len( set ) result.append( ( key, length) ) for key in t2.keys(): set = t2[ key ] if type( set ) is IntType: length = 1 else: length = len( set ) result.append( ( key, length) ) return tuple( result ) def _apply_index( self, request, cid='' ): """ Apply the index to query parameters given in 'request', which should be a mapping object. If the request does not contain the needed parametrs, then return None. If the request contains a parameter with the name of the column + "_usage", snif for information on how to handle applying the index. Otherwise return two objects. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ record = parseIndexRequest( request, self.getId() ) if record.keys is None: return None term = self._convertDateTime( record.keys[0] ) # # Aggregate sets for each bucket separately, to avoid # large-small union penalties. # #until_only = IISet() #map( until_only.update, self._until_only.values( term ) ) # XXX use multi-union until_only = multiunion( self._until_only.values( term ) ) #since_only = IISet() #map( since_only.update, self._since_only.values( None, term ) ) # XXX use multi-union since_only = multiunion( self._since_only.values( None, term ) ) #until = IISet() #map( until.update, self._until.values( term ) ) # XXX use multi-union until = multiunion( self._until.values( term ) ) #since = IISet() #map( since.update, self._since.values( None, term ) ) # XXX use multi-union since = multiunion( self._since.values( None, term ) ) bounded = intersection( until, since ) # Merge from smallest to largest. #result = union( self._always, until_only ) result = union( bounded, until_only ) result = union( result, since_only ) #result = union( result, bounded ) result = union( result, self._always ) return result, ( self._since_field, self._until_field ) # # ZCatalog needs this, although it isn't (yet) part of the interface. # security.declareProtected( VIEW_PERMISSION , 'numObjects' ) def numObjects( self ): """ """ return len( self._unindex ) # # Helper functions. # def _insertForwardIndexEntry( self, since, until, documentId ): """ Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert( documentId ) elif since is None: set = self._until_only.get( until, None ) if set is None: set = self._until_only[ until ] = IISet() # XXX: Store an int? set.insert( documentId ) elif until is None: set = self._since_only.get( since, None ) if set is None: set = self._since_only[ since ] = IISet() # XXX: Store an int? set.insert( documentId ) else: set = self._since.get( since, None ) if set is None: set = self._since[ since ] = IISet() # XXX: Store an int? set.insert( documentId ) set = self._until.get( until, None ) if set is None: set = self._until[ until ] = IISet() # XXX: Store an int? set.insert( documentId ) def _removeForwardIndexEntry( self, since, until, documentId ): """ Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove( documentId ) elif since is None: set = self._until_only.get( until, None ) if set is not None: set.remove( documentId ) if not set: del self._until_only[ until ] elif until is None: set = self._since_only.get( since, None ) if set is not None: set.remove( documentId ) if not set: del self._since_only[ since ] else: set = self._since.get( since, None ) if set is not None: set.remove( documentId ) if not set: del self._since[ since ] set = self._until.get( until, None ) if set is not None: set.remove( documentId ) if not set: del self._until[ until ] def _convertDateTime( self, value ): if value is None: return value if type( value ) == type( '' ): dt_obj = DateTime( value ) value = dt_obj.millis() / 1000 / 60 # flatten to minutes if isinstance( value, DateTime ): value = value.millis() / 1000 / 60 # flatten to minutes return int( value )
class BooleanIndex(UnIndex): """Index for booleans self._index = set([documentId1, documentId2]) self._unindex = {documentId:[True/False]} False doesn't have actual entries in _index. """ meta_type = "BooleanIndex" manage_options = ( { 'label': 'Settings', 'action': 'manage_main' }, { 'label': 'Browse', 'action': 'manage_browse' }, ) query_options = ["query"] manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals()) manage_main._setName('manage_main') manage_browse = DTMLFile('../dtml/browseIndex', globals()) def clear(self): self._length = BTrees.Length.Length() self._index = IITreeSet() self._unindex = IIBTree() def insertForwardIndexEntry(self, entry, documentId): """If True, insert directly into treeset """ if entry: self._index.insert(documentId) self._length.change(1) def removeForwardIndexEntry(self, entry, documentId): """Take the entry provided and remove any reference to documentId in its entry in the index. """ try: if entry: self._index.remove(documentId) self._length.change(-1) except ConflictError: raise except Exception: LOG.exception( '%s: unindex_object could not remove ' 'documentId %s from index %s. This ' 'should not happen.' % (self.__class__.__name__, str(documentId), str(self.id))) def _index_object(self, documentId, obj, threshold=None, attr=''): """ index and object 'obj' with integer id 'documentId'""" returnStatus = 0 # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj, attr) # Make it boolean, int as an optimization if datum is not _marker: datum = int(bool(datum)) # We don't want to do anything that we don't have to here, so we'll # check to see if the new and existing information is the same. oldDatum = self._unindex.get(documentId, _marker) if datum != oldDatum: if oldDatum is not _marker: self.removeForwardIndexEntry(oldDatum, documentId) if datum is _marker: try: del self._unindex[documentId] except ConflictError: raise except Exception: LOG.error('Should not happen: oldDatum was there, now ' 'its not, for document with id %s' % documentId) if datum is not _marker: if datum: self.insertForwardIndexEntry(datum, documentId) self._unindex[documentId] = datum returnStatus = 1 return returnStatus def _apply_index(self, request, resultset=None): record = parseIndexRequest(request, self.id, self.query_options) if record.keys is None: return None index = self._index for key in record.keys: if key: # If True, check index return (intersection(index, resultset), (self.id, )) else: # Otherwise, remove from resultset or _unindex if resultset is None: return (union(difference(self._unindex, index), IISet([])), (self.id, )) else: return (difference(resultset, index), (self.id, )) return (IISet(), (self.id, )) def indexSize(self): """Return distinct values, as an optimization we always claim 2.""" return 2 def items(self): items = [] for v, k in self._unindex.items(): if isinstance(v, int): v = IISet((v, )) items.append((k, v)) return items
class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match (i.e., they returned None for both); - Objects which match after a given time (i.e., they returned None for the end date); - Objects which match until a given time (i.e., they returned None for the start date); - Objects which match only during a specific interval. """ security = ClassSecurityInfo() meta_type = "DateRangeIndex" query_options = ('query', ) manage_options = ({'label': 'Properties', 'action': 'manage_indexProperties'}, ) since_field = until_field = None # int(DateTime('1000/1/1 0:00 GMT-12').millis() / 1000 / 60) floor_value = -510162480 # int(DateTime('2499/12/31 0:00 GMT+12').millis() / 1000 / 60) ceiling_value = 278751600 def __init__(self, id, since_field=None, until_field=None, caller=None, extra=None, floor_value=None, ceiling_value=None): if extra: since_field = extra.since_field until_field = extra.until_field floor_value = getattr(extra, 'floor_value', None) ceiling_value = getattr(extra, 'ceiling_value', None) self._setId(id) self._edit(since_field, until_field, floor_value, ceiling_value) self.clear() security.declareProtected(view, 'getSinceField') def getSinceField(self): """Get the name of the attribute indexed as start date. """ return self._since_field security.declareProtected(view, 'getUntilField') def getUntilField(self): """Get the name of the attribute indexed as end date. """ return self._until_field security.declareProtected(view, 'getFloorValue') def getFloorValue(self): """ """ return self.floor_value security.declareProtected(view, 'getCeilingValue') def getCeilingValue(self): """ """ return self.ceiling_value manage_indexProperties = DTMLFile('manageDateRangeIndex', _dtmldir) security.declareProtected(manage_zcatalog_indexes, 'manage_edit') def manage_edit(self, since_field, until_field, floor_value, ceiling_value, REQUEST): """ """ self._edit(since_field, until_field, floor_value, ceiling_value) REQUEST['RESPONSE'].redirect('%s/manage_main' '?manage_tabs_message=Updated' % REQUEST.get('URL2')) security.declarePrivate('_edit') def _edit(self, since_field, until_field, floor_value=None, ceiling_value=None): """Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field if floor_value is not None: self.floor_value = int(floor_value) if ceiling_value is not None: self.ceiling_value = int(ceiling_value) security.declareProtected(manage_zcatalog_indexes, 'clear') def clear(self): """Start over fresh.""" self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length() if self._counter is None: self._counter = Length() else: self._increment_counter() def getEntryForObject(self, documentId, default=None): """Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get(documentId, default) def index_object(self, documentId, obj, threshold=None): """Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr(obj, self._since_field, None) if safe_callable(since): since = since() since = self._convertDateTime(since) until = getattr(obj, self._until_field, None) if safe_callable(until): until = until() until = self._convertDateTime(until) datum = (since, until) old_datum = self._unindex.get(documentId, None) if datum == old_datum: # No change? bail out! return 0 self._increment_counter() if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry(old_since, old_until, documentId) self._insertForwardIndexEntry(since, until, documentId) self._unindex[documentId] = datum return 1 def unindex_object(self, documentId): """Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get(documentId, None) if datum is None: return self._increment_counter() since, until = datum self._removeForwardIndexEntry(since, until, documentId) del self._unindex[documentId] def uniqueValues(self, name=None, withLengths=0): """Return a sequence of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '(value, length)'. """ if name not in (self._since_field, self._until_field): raise StopIteration if name == self._since_field: sets = (self._since, self._since_only) else: sets = (self._until, self._until_only) if not withLengths: for s in sets: for key in s.keys(): yield key else: for s in sets: for key, value in s.items(): if isinstance(value, int): yield (key, 1) else: yield (key, len(value)) def getRequestCacheKey(self, record, resultset=None): term = self._convertDateTime(record.keys[0]) tid = str(term) # unique index identifier iid = '_%s_%s_%s' % (self.__class__.__name__, self.id, self.getCounter()) # record identifier if resultset is None: rid = '_%s' % (tid, ) else: rid = '_inverse_%s' % (tid, ) return (iid, rid) def _apply_index(self, request, resultset=None): record = IndexQuery(request, self.id, self.query_options, self.operators, self.useOperator) if record.keys is None: return None return (self.query_index(record, resultset=resultset), (self._since_field, self._until_field)) def query_index(self, record, resultset=None): cache = self.getRequestCache() if cache is not None: cachekey = self.getRequestCacheKey(record, resultset) cached = cache.get(cachekey, None) if cached is not None: if resultset is None: return cached else: return difference(resultset, cached) term = self._convertDateTime(record.keys[0]) if resultset is None: # Aggregate sets for each bucket separately, to avoid # large-small union penalties. until_only = multiunion(self._until_only.values(term)) since_only = multiunion(self._since_only.values(None, term)) until = multiunion(self._until.values(term)) since = multiunion(self._since.values(None, term)) bounded = intersection(until, since) # Merge from smallest to largest. result = multiunion([bounded, until_only, since_only, self._always]) if cache is not None: cache[cachekey] = result return result else: # Compute the inverse and subtract from res until_only = multiunion(self._until_only.values(None, term - 1)) since_only = multiunion(self._since_only.values(term + 1)) until = multiunion(self._until.values(None, term - 1)) since = multiunion(self._since.values(term + 1)) result = multiunion([since, since_only, until_only, until]) if cache is not None: cache[cachekey] = result return difference(resultset, result) def _insert_migrate(self, tree, key, value): treeset = tree.get(key, None) if treeset is None: tree[key] = IITreeSet((value, )) else: if isinstance(treeset, IITreeSet): treeset.insert(value) elif isinstance(treeset, int): tree[key] = IITreeSet((treeset, value)) else: tree[key] = IITreeSet(treeset) tree[key].insert(value) def _insertForwardIndexEntry(self, since, until, documentId): """Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert(documentId) elif since is None: self._insert_migrate(self._until_only, until, documentId) elif until is None: self._insert_migrate(self._since_only, since, documentId) else: self._insert_migrate(self._since, since, documentId) self._insert_migrate(self._until, until, documentId) def _remove_delete(self, tree, key, value): treeset = tree.get(key, None) if treeset is not None: if isinstance(treeset, int): del tree[key] else: treeset.remove(value) if not treeset: del tree[key] def _removeForwardIndexEntry(self, since, until, documentId): """Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove(documentId) elif since is None: self._remove_delete(self._until_only, until, documentId) elif until is None: self._remove_delete(self._since_only, since, documentId) else: self._remove_delete(self._since, since, documentId) self._remove_delete(self._until, until, documentId) def _convertDateTime(self, value): if value is None: return value if isinstance(value, (str, datetime)): dt_obj = DateTime(value) value = dt_obj.millis() / 1000 / 60 # flatten to minutes elif isinstance(value, DateTime): value = value.millis() / 1000 / 60 # flatten to minutes if value > MAX32 or value < -MAX32: # t_val must be integer fitting in the 32bit range raise OverflowError('%s is not within the range of dates allowed' 'by a DateRangeIndex' % value) value = int(value) # handle values outside our specified range if value > self.ceiling_value: return None elif value < self.floor_value: return None return value
def remove(self, obj): return IITreeSet.remove(self, self._get_id(obj))
class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match (i.e., they returned None for both); - Objects which match after a given time (i.e., they returned None for the end date); - Objects which match until a given time (i.e., they returned None for the start date); - Objects which match only during a specific interval. """ security = ClassSecurityInfo() meta_type = 'DateRangeIndex' query_options = ('query', ) manage_options = ({ 'label': 'Properties', 'action': 'manage_indexProperties' }, ) since_field = until_field = None # int(DateTime('1000/1/1 0:00 GMT-12').millis() / 1000 / 60) floor_value = -510162480 # int(DateTime('2499/12/31 0:00 GMT+12').millis() / 1000 / 60) ceiling_value = 278751600 # precision of indexed time interval in minutes precision_value = 1 def __init__(self, id, since_field=None, until_field=None, caller=None, extra=None, floor_value=None, ceiling_value=None, precision_value=None): if extra: since_field = extra.since_field until_field = extra.until_field floor_value = getattr(extra, 'floor_value', None) ceiling_value = getattr(extra, 'ceiling_value', None) precision_value = getattr(extra, 'precision_value', None) self._setId(id) self._edit(since_field, until_field, floor_value, ceiling_value, precision_value) self.clear() @security.protected(view) def getSinceField(self): """Get the name of the attribute indexed as start date. """ return self._since_field @security.protected(view) def getUntilField(self): """Get the name of the attribute indexed as end date. """ return self._until_field @security.protected(view) def getFloorValue(self): """ """ return self.floor_value @security.protected(view) def getCeilingValue(self): """ """ return self.ceiling_value @security.protected(view) def getPrecisionValue(self): """ """ return self.precision_value manage_indexProperties = DTMLFile('manageDateRangeIndex', _dtmldir) @security.protected(manage_zcatalog_indexes) def manage_edit(self, since_field, until_field, floor_value, ceiling_value, precision_value, REQUEST): """ """ self._edit(since_field, until_field, floor_value, ceiling_value, precision_value) REQUEST['RESPONSE'].redirect('{0}/manage_main' '?manage_tabs_message=Updated'.format( REQUEST.get('URL2'))) @security.private def _edit(self, since_field, until_field, floor_value=None, ceiling_value=None, precision_value=None): """Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field if floor_value not in (None, ''): self.floor_value = int(floor_value) if ceiling_value not in (None, ''): self.ceiling_value = int(ceiling_value) if precision_value not in (None, ''): self.precision_value = int(precision_value) @security.protected(manage_zcatalog_indexes) def clear(self): """Start over fresh.""" self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length() if self._counter is None: self._counter = Length() else: self._increment_counter() def getEntryForObject(self, documentId, default=None): """Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get(documentId, default) def index_object(self, documentId, obj, threshold=None): """Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr(obj, self._since_field, None) if safe_callable(since): since = since() since = self._convertDateTime(since) until = getattr(obj, self._until_field, None) if safe_callable(until): until = until() until = self._convertDateTime(until) datum = (since, until) old_datum = self._unindex.get(documentId, None) if datum == old_datum: # No change? bail out! return 0 self._increment_counter() if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry(old_since, old_until, documentId) self._insertForwardIndexEntry(since, until, documentId) self._unindex[documentId] = datum return 1 def unindex_object(self, documentId): """Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get(documentId, None) if datum is None: return self._increment_counter() since, until = datum self._removeForwardIndexEntry(since, until, documentId) del self._unindex[documentId] def uniqueValues(self, name=None, withLengths=0): """Return a sequence of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '(value, length)'. """ if name not in (self._since_field, self._until_field): return if name == self._since_field: sets = (self._since, self._since_only) else: sets = (self._until, self._until_only) if not withLengths: for s in sets: for key in s.keys(): yield key else: for s in sets: for key, value in s.items(): if isinstance(value, int): yield (key, 1) else: yield (key, len(value)) def getRequestCacheKey(self, record, resultset=None): term = self._convertDateTime(record.keys[0]) tid = str(term) # unique index identifier iid = '_{0}_{1}_{2}'.format(self.__class__.__name__, self.id, self.getCounter()) # record identifier if resultset is None: rid = '_{0}'.format(tid) else: rid = '_inverse_{0}'.format(tid) return (iid, rid) def _apply_index(self, request, resultset=None): record = IndexQuery(request, self.id, self.query_options, self.operators, self.useOperator) if record.keys is None: return None return (self.query_index(record, resultset=resultset), (self._since_field, self._until_field)) def query_index(self, record, resultset=None): cache = self.getRequestCache() if cache is not None: cachekey = self.getRequestCacheKey(record, resultset) cached = cache.get(cachekey, None) if cached is not None: if resultset is None: return cached else: return difference(resultset, cached) term = self._convertDateTime(record.keys[0]) if resultset is None: # Aggregate sets for each bucket separately, to avoid # large-small union penalties. until_only = multiunion(self._until_only.values(term)) since_only = multiunion(self._since_only.values(None, term)) until = multiunion(self._until.values(term)) since = multiunion(self._since.values(None, term)) bounded = intersection(until, since) # Merge from smallest to largest. result = multiunion( [bounded, until_only, since_only, self._always]) if cache is not None: cache[cachekey] = result return result else: # Compute the inverse and subtract from res until_only = multiunion(self._until_only.values(None, term - 1)) since_only = multiunion(self._since_only.values(term + 1)) until = multiunion(self._until.values(None, term - 1)) since = multiunion(self._since.values(term + 1)) result = multiunion([since, since_only, until_only, until]) if cache is not None: cache[cachekey] = result return difference(resultset, result) def _insert_migrate(self, tree, key, value): treeset = tree.get(key, None) if treeset is None: tree[key] = IITreeSet((value, )) else: if isinstance(treeset, IITreeSet): treeset.insert(value) elif isinstance(treeset, int): tree[key] = IITreeSet((treeset, value)) else: tree[key] = IITreeSet(treeset) tree[key].insert(value) def _insertForwardIndexEntry(self, since, until, documentId): """Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert(documentId) elif since is None: self._insert_migrate(self._until_only, until, documentId) elif until is None: self._insert_migrate(self._since_only, since, documentId) else: self._insert_migrate(self._since, since, documentId) self._insert_migrate(self._until, until, documentId) def _remove_delete(self, tree, key, value): treeset = tree.get(key, None) if treeset is not None: if isinstance(treeset, int): del tree[key] else: treeset.remove(value) if not treeset: del tree[key] def _removeForwardIndexEntry(self, since, until, documentId): """Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove(documentId) elif since is None: self._remove_delete(self._until_only, until, documentId) elif until is None: self._remove_delete(self._since_only, since, documentId) else: self._remove_delete(self._since, since, documentId) self._remove_delete(self._until, until, documentId) def _convertDateTime(self, value): value = datetime_to_minutes(value, self.precision_value) if value is None: return None if (value > self.ceiling_value or value < self.floor_value): # handle values outside our specified range return None return value
class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match (i.e., they returned None for both); - Objects which match after a given time (i.e., they returned None for the end date); - Objects which match until a given time (i.e., they returned None for the start date); - Objects which match only during a specific interval. """ implements(IDateRangeIndex) security = ClassSecurityInfo() meta_type = "DateRangeIndex" query_options = ("query",) manage_options = ({"label": "Properties", "action": "manage_indexProperties"},) since_field = until_field = None # int(DateTime('1000/1/1 0:00 GMT-12').millis() / 1000 / 60) floor_value = -510162480 # int(DateTime('2499/12/31 0:00 GMT+12').millis() / 1000 / 60) ceiling_value = 278751600 def __init__( self, id, since_field=None, until_field=None, caller=None, extra=None, floor_value=None, ceiling_value=None ): if extra: since_field = extra.since_field until_field = extra.until_field floor_value = getattr(extra, "floor_value", None) ceiling_value = getattr(extra, "ceiling_value", None) self._setId(id) self._edit(since_field, until_field, floor_value, ceiling_value) self.clear() security.declareProtected(view, "getSinceField") def getSinceField(self): """Get the name of the attribute indexed as start date. """ return self._since_field security.declareProtected(view, "getUntilField") def getUntilField(self): """Get the name of the attribute indexed as end date. """ return self._until_field security.declareProtected(view, "getFloorValue") def getFloorValue(self): """""" return self.floor_value security.declareProtected(view, "getCeilingValue") def getCeilingValue(self): """""" return self.ceiling_value manage_indexProperties = DTMLFile("manageDateRangeIndex", _dtmldir) security.declareProtected(manage_zcatalog_indexes, "manage_edit") def manage_edit(self, since_field, until_field, floor_value, ceiling_value, REQUEST): """""" self._edit(since_field, until_field, floor_value, ceiling_value) REQUEST["RESPONSE"].redirect("%s/manage_main" "?manage_tabs_message=Updated" % REQUEST.get("URL2")) security.declarePrivate("_edit") def _edit(self, since_field, until_field, floor_value=None, ceiling_value=None): """Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field if floor_value is not None: self.floor_value = int(floor_value) if ceiling_value is not None: self.ceiling_value = int(ceiling_value) security.declareProtected(manage_zcatalog_indexes, "clear") def clear(self): """Start over fresh.""" self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length() def getEntryForObject(self, documentId, default=None): """Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get(documentId, default) def index_object(self, documentId, obj, threshold=None): """Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr(obj, self._since_field, None) if safe_callable(since): since = since() since = self._convertDateTime(since) until = getattr(obj, self._until_field, None) if safe_callable(until): until = until() until = self._convertDateTime(until) datum = (since, until) old_datum = self._unindex.get(documentId, None) if datum == old_datum: # No change? bail out! return 0 if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry(old_since, old_until, documentId) self._insertForwardIndexEntry(since, until, documentId) self._unindex[documentId] = datum return 1 def unindex_object(self, documentId): """Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get(documentId, None) if datum is None: return since, until = datum self._removeForwardIndexEntry(since, until, documentId) del self._unindex[documentId] def uniqueValues(self, name=None, withLengths=0): """Return a list of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '(value, length)'. """ if not name in (self._since_field, self._until_field): return [] if name == self._since_field: t1 = self._since t2 = self._since_only else: t1 = self._until t2 = self._until_only result = [] if not withLengths: result.extend(t1.keys()) result.extend(t2.keys()) else: for key in t1.keys(): set = t1[key] if isinstance(set, int): length = 1 else: length = len(set) result.append((key, length)) for key in t2.keys(): set = t2[key] if isinstance(set, int): length = 1 else: length = len(set) result.append((key, length)) return tuple(result) def _cache_key(self, catalog): cid = catalog.getId() counter = getattr(aq_base(catalog), "getCounter", None) if counter is not None: return "%s_%s" % (cid, counter()) return cid def _apply_index(self, request, resultset=None): """Apply the index to query parameters given in 'request', which should be a mapping object. If the request does not contain the needed parameters, then return None. Otherwise return two objects. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ iid = self.id record = parseIndexRequest(request, iid, self.query_options) if record.keys is None: return None term = self._convertDateTime(record.keys[0]) REQUEST = aq_get(self, "REQUEST", None) if REQUEST is not None: catalog = aq_parent(aq_parent(aq_inner(self))) if catalog is not None: key = self._cache_key(catalog) cache = REQUEST.get(key, None) tid = isinstance(term, int) and term / 10 or "None" if resultset is None: cachekey = "_daterangeindex_%s_%s" % (iid, tid) else: cachekey = "_daterangeindex_inverse_%s_%s" % (iid, tid) if cache is None: cache = REQUEST[key] = RequestCache() else: cached = cache.get(cachekey, None) if cached is not None: if resultset is None: return (cached, (self._since_field, self._until_field)) else: return (difference(resultset, cached), (self._since_field, self._until_field)) if resultset is None: # Aggregate sets for each bucket separately, to avoid # large-small union penalties. until_only = multiunion(self._until_only.values(term)) since_only = multiunion(self._since_only.values(None, term)) until = multiunion(self._until.values(term)) # Total result is bound by resultset if REQUEST is None: until = intersection(resultset, until) since = multiunion(self._since.values(None, term)) bounded = intersection(until, since) # Merge from smallest to largest. result = multiunion([bounded, until_only, since_only, self._always]) if REQUEST is not None and catalog is not None: cache[cachekey] = result return (result, (self._since_field, self._until_field)) else: # Compute the inverse and subtract from res until_only = multiunion(self._until_only.values(None, term - 1)) since_only = multiunion(self._since_only.values(term + 1)) until = multiunion(self._until.values(None, term - 1)) since = multiunion(self._since.values(term + 1)) result = multiunion([since, since_only, until_only, until]) if REQUEST is not None and catalog is not None: cache[cachekey] = result return (difference(resultset, result), (self._since_field, self._until_field)) def _insert_migrate(self, tree, key, value): treeset = tree.get(key, None) if treeset is None: tree[key] = IITreeSet((value,)) else: if isinstance(treeset, IITreeSet): treeset.insert(value) elif isinstance(treeset, int): tree[key] = IITreeSet((treeset, value)) else: tree[key] = IITreeSet(treeset) tree[key].insert(value) def _insertForwardIndexEntry(self, since, until, documentId): """Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert(documentId) elif since is None: self._insert_migrate(self._until_only, until, documentId) elif until is None: self._insert_migrate(self._since_only, since, documentId) else: self._insert_migrate(self._since, since, documentId) self._insert_migrate(self._until, until, documentId) def _remove_delete(self, tree, key, value): treeset = tree.get(key, None) if treeset is not None: if isinstance(treeset, int): del tree[key] else: treeset.remove(value) if not treeset: del tree[key] def _removeForwardIndexEntry(self, since, until, documentId): """Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove(documentId) elif since is None: self._remove_delete(self._until_only, until, documentId) elif until is None: self._remove_delete(self._since_only, since, documentId) else: self._remove_delete(self._since, since, documentId) self._remove_delete(self._until, until, documentId) def _convertDateTime(self, value): if value is None: return value if isinstance(value, (str, datetime)): dt_obj = DateTime(value) value = dt_obj.millis() / 1000 / 60 # flatten to minutes elif isinstance(value, DateTime): value = value.millis() / 1000 / 60 # flatten to minutes if value > MAX32 or value < -MAX32: # t_val must be integer fitting in the 32bit range raise OverflowError("%s is not within the range of dates allowed" "by a DateRangeIndex" % value) value = int(value) # handle values outside our specified range if value > self.ceiling_value: return None elif value < self.floor_value: return None return value
def remove(self, obj): return IITreeSet.remove(self, self._get_id(obj))
class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. Any object may return None for either the start or the end date: for the start date, this should be the logical equivalent of "since the beginning of time"; for the end date, "until the end of time". Therefore, divide the space of indexed objects into four containers: - Objects which always match (i.e., they returned None for both); - Objects which match after a given time (i.e., they returned None for the end date); - Objects which match until a given time (i.e., they returned None for the start date); - Objects which match only during a specific interval. """ __implements__ = UnIndex.__implements__ implements(IDateRangeIndex) security = ClassSecurityInfo() meta_type = "DateRangeIndex" manage_options = ({ 'label': 'Properties', 'action': 'manage_indexProperties' }, ) query_options = ['query'] since_field = until_field = None def __init__(self, id, since_field=None, until_field=None, caller=None, extra=None): if extra: since_field = extra.since_field until_field = extra.until_field self._setId(id) self._edit(since_field, until_field) self.clear() security.declareProtected(VIEW_PERMISSION, 'getSinceField') def getSinceField(self): """Get the name of the attribute indexed as start date. """ return self._since_field security.declareProtected(VIEW_PERMISSION, 'getUntilField') def getUntilField(self): """Get the name of the attribute indexed as end date. """ return self._until_field manage_indexProperties = DTMLFile('manageDateRangeIndex', _dtmldir) security.declareProtected(INDEX_MGMT_PERMISSION, 'manage_edit') def manage_edit(self, since_field, until_field, REQUEST): """ """ self._edit(since_field, until_field) REQUEST['RESPONSE'].redirect('%s/manage_main' '?manage_tabs_message=Updated' % REQUEST.get('URL2')) security.declarePrivate('_edit') def _edit(self, since_field, until_field): """ Update the fields used to compute the range. """ self._since_field = since_field self._until_field = until_field security.declareProtected(INDEX_MGMT_PERMISSION, 'clear') def clear(self): """ Start over fresh. """ self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = BTrees.Length.Length() # # PluggableIndexInterface implementation (XXX inherit assertions?) # def getEntryForObject(self, documentId, default=None): """ Get all information contained for the specific object identified by 'documentId'. Return 'default' if not found. """ return self._unindex.get(documentId, default) def index_object(self, documentId, obj, threshold=None): """ Index an object: - 'documentId' is the integer ID of the document - 'obj' is the object to be indexed - ignore threshold """ if self._since_field is None: return 0 since = getattr(obj, self._since_field, None) if safe_callable(since): since = since() since = self._convertDateTime(since) until = getattr(obj, self._until_field, None) if safe_callable(until): until = until() until = self._convertDateTime(until) datum = (since, until) old_datum = self._unindex.get(documentId, None) if datum == old_datum: # No change? bail out! return 0 if old_datum is not None: old_since, old_until = old_datum self._removeForwardIndexEntry(old_since, old_until, documentId) self._insertForwardIndexEntry(since, until, documentId) self._unindex[documentId] = datum return 1 def unindex_object(self, documentId): """ Remove the object corresponding to 'documentId' from the index. """ datum = self._unindex.get(documentId, None) if datum is None: return since, until = datum self._removeForwardIndexEntry(since, until, documentId) del self._unindex[documentId] def uniqueValues(self, name=None, withLengths=0): """ Return a list of unique values for 'name'. If 'withLengths' is true, return a sequence of tuples, in the form '( value, length )'. """ if not name in (self._since_field, self._until_field): return [] if name == self._since_field: t1 = self._since t2 = self._since_only else: t1 = self._until t2 = self._until_only result = [] IntType = type(0) if not withLengths: result.extend(t1.keys()) result.extend(t2.keys()) else: for key in t1.keys(): set = t1[key] if type(set) is IntType: length = 1 else: length = len(set) result.append((key, length)) for key in t2.keys(): set = t2[key] if type(set) is IntType: length = 1 else: length = len(set) result.append((key, length)) return tuple(result) def _apply_index(self, request, cid=''): """ Apply the index to query parameters given in 'request', which should be a mapping object. If the request does not contain the needed parametrs, then return None. If the request contains a parameter with the name of the column + "_usage", snif for information on how to handle applying the index. Otherwise return two objects. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ record = parseIndexRequest(request, self.getId()) if record.keys is None: return None term = self._convertDateTime(record.keys[0]) # # Aggregate sets for each bucket separately, to avoid # large-small union penalties. # #until_only = IISet() #map( until_only.update, self._until_only.values( term ) ) # XXX use multi-union until_only = multiunion(self._until_only.values(term)) #since_only = IISet() #map( since_only.update, self._since_only.values( None, term ) ) # XXX use multi-union since_only = multiunion(self._since_only.values(None, term)) #until = IISet() #map( until.update, self._until.values( term ) ) # XXX use multi-union until = multiunion(self._until.values(term)) #since = IISet() #map( since.update, self._since.values( None, term ) ) # XXX use multi-union since = multiunion(self._since.values(None, term)) bounded = intersection(until, since) # Merge from smallest to largest. #result = union( self._always, until_only ) result = union(bounded, until_only) result = union(result, since_only) #result = union( result, bounded ) result = union(result, self._always) return result, (self._since_field, self._until_field) # # ZCatalog needs this, although it isn't (yet) part of the interface. # security.declareProtected(VIEW_PERMISSION, 'numObjects') def numObjects(self): """ """ return len(self._unindex) def indexSize(self): """ """ return len(self) # # Helper functions. # def _insertForwardIndexEntry(self, since, until, documentId): """ Insert 'documentId' into the appropriate set based on 'datum'. """ if since is None and until is None: self._always.insert(documentId) elif since is None: set = self._until_only.get(until, None) if set is None: set = self._until_only[until] = IISet() # XXX: Store an int? set.insert(documentId) elif until is None: set = self._since_only.get(since, None) if set is None: set = self._since_only[since] = IISet() # XXX: Store an int? set.insert(documentId) else: set = self._since.get(since, None) if set is None: set = self._since[since] = IISet() # XXX: Store an int? set.insert(documentId) set = self._until.get(until, None) if set is None: set = self._until[until] = IISet() # XXX: Store an int? set.insert(documentId) def _removeForwardIndexEntry(self, since, until, documentId): """ Remove 'documentId' from the appropriate set based on 'datum'. """ if since is None and until is None: self._always.remove(documentId) elif since is None: set = self._until_only.get(until, None) if set is not None: set.remove(documentId) if not set: del self._until_only[until] elif until is None: set = self._since_only.get(since, None) if set is not None: set.remove(documentId) if not set: del self._since_only[since] else: set = self._since.get(since, None) if set is not None: set.remove(documentId) if not set: del self._since[since] set = self._until.get(until, None) if set is not None: set.remove(documentId) if not set: del self._until[until] def _convertDateTime(self, value): if value is None: return value if type(value) == type(''): dt_obj = DateTime(value) value = dt_obj.millis() / 1000 / 60 # flatten to minutes if isinstance(value, DateTime): value = value.millis() / 1000 / 60 # flatten to minutes result = int(value) if isinstance(result, long): # this won't work (Python 2.3) raise OverflowError('%s is not within the range of dates allowed' 'by a DateRangeIndex' % value) return result