class DeployedStore(Persistent): """ Deployed entries storage """ implements(IDeployedStore) store_length = 10 def __init__(self): self._entries = OOBTree() self.busy = False def __iter__(self): return reversed([i for i in self._entries.itervalues()]) def add(self, date, user, action, clear, full, status, errmsg=None): entry = DeployedEntry(date, user, action, clear, full, status, errmsg) self._entries[entry.id] = entry if len(self._entries) > self.store_length: del self._entries[self._entries.minKey()] def remove(self, id): del self._entries[id] def get(self, id): return self._entries[id] @iteration_with_status(DEPLOYMENT_STATUS.DONE) def done(self): pass @iteration_with_status(DEPLOYMENT_STATUS.ERROR) def error(self): pass
class fsIndex(object): def __init__(self, data=None): self._data = OOBTree() if data: self.update(data) def __getstate__(self): return dict(state_version=1, _data=[(k, v.toString()) for (k, v) in self._data.iteritems()]) def __setstate__(self, state): version = state.pop("state_version", 0) getattr(self, "_setstate_%s" % version)(state) def _setstate_0(self, state): self.__dict__.clear() self.__dict__.update(state) def _setstate_1(self, state): self._data = OOBTree([(k, fsBucket().fromString(v)) for (k, v) in state["_data"]]) def __getitem__(self, key): return str2num(self._data[key[:6]][key[6:]]) def save(self, pos, fname): with open(fname, "wb") as f: pickler = cPickle.Pickler(f, 1) pickler.fast = True pickler.dump(pos) for k, v in self._data.iteritems(): pickler.dump((k, v.toString())) pickler.dump(None) @classmethod def load(class_, fname): with open(fname, "rb") as f: unpickler = cPickle.Unpickler(f) pos = unpickler.load() if not isinstance(pos, (int, long)): return pos # Old format index = class_() data = index._data while 1: v = unpickler.load() if not v: break k, v = v data[k] = fsBucket().fromString(v) return dict(pos=pos, index=index) def get(self, key, default=None): tree = self._data.get(key[:6], default) if tree is default: return default v = tree.get(key[6:], default) if v is default: return default return str2num(v) def __setitem__(self, key, value): value = num2str(value) treekey = key[:6] tree = self._data.get(treekey) if tree is None: tree = fsBucket() self._data[treekey] = tree tree[key[6:]] = value def __delitem__(self, key): treekey = key[:6] tree = self._data.get(treekey) if tree is None: raise KeyError, key del tree[key[6:]] if not tree: del self._data[treekey] def __len__(self): r = 0 for tree in self._data.itervalues(): r += len(tree) return r def update(self, mapping): for k, v in mapping.items(): self[k] = v def has_key(self, key): v = self.get(key, self) return v is not self def __contains__(self, key): tree = self._data.get(key[:6]) if tree is None: return False v = tree.get(key[6:], None) if v is None: return False return True def clear(self): self._data.clear() def __iter__(self): for prefix, tree in self._data.iteritems(): for suffix in tree: yield prefix + suffix iterkeys = __iter__ def keys(self): return list(self.iterkeys()) def iteritems(self): for prefix, tree in self._data.iteritems(): for suffix, value in tree.iteritems(): yield (prefix + suffix, str2num(value)) def items(self): return list(self.iteritems()) def itervalues(self): for tree in self._data.itervalues(): for value in tree.itervalues(): yield str2num(value) def values(self): return list(self.itervalues()) # Comment below applies for the following minKey and maxKey methods # # Obscure: what if `tree` is actually empty? We're relying here on # that this class doesn't implement __delitem__: once a key gets # into an fsIndex, the only way it can go away is by invoking # clear(). Therefore nothing in _data.values() is ever empty. # # Note that because `tree` is an fsBTree, its minKey()/maxKey() methods are # very efficient. def minKey(self, key=None): if key is None: smallest_prefix = self._data.minKey() else: smallest_prefix = self._data.minKey(key[:6]) tree = self._data[smallest_prefix] assert tree if key is None: smallest_suffix = tree.minKey() else: try: smallest_suffix = tree.minKey(key[6:]) except ValueError: # 'empty tree' (no suffix >= arg) next_prefix = prefix_plus_one(smallest_prefix) smallest_prefix = self._data.minKey(next_prefix) tree = self._data[smallest_prefix] assert tree smallest_suffix = tree.minKey() return smallest_prefix + smallest_suffix def maxKey(self, key=None): if key is None: biggest_prefix = self._data.maxKey() else: biggest_prefix = self._data.maxKey(key[:6]) tree = self._data[biggest_prefix] assert tree if key is None: biggest_suffix = tree.maxKey() else: try: biggest_suffix = tree.maxKey(key[6:]) except ValueError: # 'empty tree' (no suffix <= arg) next_prefix = prefix_minus_one(biggest_prefix) biggest_prefix = self._data.maxKey(next_prefix) tree = self._data[biggest_prefix] assert tree biggest_suffix = tree.maxKey() return biggest_prefix + biggest_suffix
class MultiPointerTrack(Persistent): """ A MultiPointerTrack is a kind of structure that is based on an IOBTree, where each entry contains an ordered set (or list, depending on the implementation) of elements. Then, several "pointers" can be created, which point to different positions of the track (very much like runners in a race track). This class is abstract, implementations should be derived. """ def __init__(self, elemContainer): self._container = OOBTree() self._pointers = PersistentMapping() self._elemContainer = elemContainer # initialize first entry #self._container[timestamp(0)] = elemContainer() def addPointer(self, pid, startPos=None): """ Registers a new pointer """ self._pointers[pid] = None if startPos: self.movePointer(pid, startPos) def removePointer(self, pid): """ Removes a pointer from the list """ del self._pointers[pid] def prepareEntry(self, ts): """ Creates an empty sub-structure (elemContainer) for a given timestamp """ self._container[timestamp(ts)] = self._elemContainer() def getCurrentPosition(self, pid): """ Returns the current entry (set/list) for a given pointer id """ currentPos = self._pointers[pid] # TODO: assertion? check? return self._container[timestamp(currentPos)] def getPointerTimestamp(self, pid): """ Gets the current 'position' of a pointer (id) """ return self._pointers[pid] def __getitem__(self, ts): """ Implements __getitem__, so that mpt[timestamp] works """ if isinstance(ts, slice): return self._getSlice(ts) else: return self._container[timestamp(ts)] def _getSlice(self, s): """ Calculates a slice of the structure (timestamp-wise) """ if s.step != None: raise TypeError('Extended slices are not accepted here') return self._container.values(s.start, s.stop) def values(self, *args): """ Return values or ranges (timestamps) of the structure """ fargs = [] for a in args: if a == None: fargs.append(None) else: fargs.append(timestamp(a)) return self._container.values(*fargs) def _append(self, ts, val): """ Should be overloaded. """ raise Exception("Unimplemented method") def add(self, intTS, value): """ Adds a value to the container corresponding to a specific timestamp """ ts = timestamp(intTS) if ts not in self._container: self.prepareEntry(intTS) self._append(ts, value) def _pointerIterator(self, pid, func, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id). Takes a function that is applied to yielded values """ return self.iterate(self._pointers[pid], till, func) def iterate(self, fromPos=None, till=None, func=(lambda x: x)): """ Generator that iterates through the data structure """ if till != None: till = timestamp(till) # negative numbers mean "last but one", "last but two", etc... if till == timestamp(-1): # most common case till = self._container.maxKey() - timestamp(1) if fromPos != None: fromPos = timestamp(fromPos) for ts, entry in self._container.iteritems(till, fromPos): if fromPos and ts == fromPos: # stop immediately if we're past fromPos raise StopIteration for elem in entry: yield func((int(ts), elem)) def mostRecentTS(self, maximum=None): """ Returns most recent timestamp in track (minimum key) If 'maximum' is provided, return it if less recent """ # check that the tree has something if len(self._container) == 0: raise EmptyTrackException() mr = self._container.minKey() if maximum: maximum = timestamp(maximum) # in timestamp logic, max() returns the oldest return max(mr, maximum) else: return mr def oldestTS(self): """ Returns least recent timestamp in track (maximum key) """ # check that the tree has something if len(self._container) == 0: raise EmptyTrackException() return self._container.maxKey() def pointerIterValues(self, pid, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id) - iterates over values """ return self._pointerIterator(pid, lambda x: x[1], till=till) def pointerIterItems(self, pid, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id) - iterates over key-value pairs (iteritems) """ return self._pointerIterator(pid, lambda x: x, till=till) def movePointer(self, pid, pos): """ Moves a given pointer (id) to a given timestamp """ if pid not in self._pointers: raise KeyError("Pointer '%s' doesn't seem to exist!" % pid) # check that the tree has something if len(self._container) == 0: raise EmptyTrackException() self._pointers[pid] = pos def __len__(self): """ Returns the number of timestamp entries """ return len(self._container) def __delitem__(self, item): """ Deletes a given timestamp entry (or range) """ self._container.__delitem__(item) def __iter__(self): """ Iterates over the whole structure, element by elements (goes inside containers) """ return self.iterate() def __contains__(self, ts): return timestamp(ts) in self._container
class fsIndex(object): def __init__(self, data=None): self._data = OOBTree() if data: self.update(data) def __getstate__(self): return dict( state_version = 1, _data = [(k, v.toString()) for (k, v) in six.iteritems(self._data) ] ) def __setstate__(self, state): version = state.pop('state_version', 0) getattr(self, '_setstate_%s' % version)(state) def _setstate_0(self, state): self.__dict__.clear() self.__dict__.update(state) self._data = OOBTree([ (ensure_bytes(k), v) for (k, v) in self._data.items() ]) def _setstate_1(self, state): self._data = OOBTree([ (ensure_bytes(k), fsBucket().fromString(ensure_bytes(v))) for (k, v) in state['_data'] ]) def __getitem__(self, key): assert isinstance(key, bytes) return str2num(self._data[key[:6]][key[6:]]) def save(self, pos, fname): with open(fname, 'wb') as f: pickler = Pickler(f, _protocol) pickler.fast = True pickler.dump(pos) for k, v in six.iteritems(self._data): pickler.dump((k, v.toString())) pickler.dump(None) @classmethod def load(class_, fname): with open(fname, 'rb') as f: unpickler = Unpickler(f) pos = unpickler.load() if not isinstance(pos, INT_TYPES): # NB: this might contain OIDs that got unpickled # into Unicode strings on Python 3; hope the caller # will pipe the result to fsIndex().update() to normalize # the keys return pos # Old format index = class_() data = index._data while 1: v = unpickler.load() if not v: break k, v = v data[ensure_bytes(k)] = fsBucket().fromString(ensure_bytes(v)) return dict(pos=pos, index=index) def get(self, key, default=None): assert isinstance(key, bytes) tree = self._data.get(key[:6], default) if tree is default: return default v = tree.get(key[6:], default) if v is default: return default return str2num(v) def __setitem__(self, key, value): assert isinstance(key, bytes) value = num2str(value) treekey = key[:6] tree = self._data.get(treekey) if tree is None: tree = fsBucket() self._data[treekey] = tree tree[key[6:]] = value def __delitem__(self, key): assert isinstance(key, bytes) treekey = key[:6] tree = self._data.get(treekey) if tree is None: raise KeyError(key) del tree[key[6:]] if not tree: del self._data[treekey] def __len__(self): r = 0 for tree in six.itervalues(self._data): r += len(tree) return r def update(self, mapping): for k, v in mapping.items(): self[ensure_bytes(k)] = v def has_key(self, key): v = self.get(key, self) return v is not self def __contains__(self, key): assert isinstance(key, bytes) tree = self._data.get(key[:6]) if tree is None: return False v = tree.get(key[6:], None) if v is None: return False return True def clear(self): self._data.clear() def __iter__(self): for prefix, tree in six.iteritems(self._data): for suffix in tree: yield prefix + suffix iterkeys = __iter__ def keys(self): return list(self.iterkeys()) def iteritems(self): for prefix, tree in six.iteritems(self._data): for suffix, value in six.iteritems(tree): yield (prefix + suffix, str2num(value)) def items(self): return list(self.iteritems()) def itervalues(self): for tree in six.itervalues(self._data): for value in six.itervalues(tree): yield str2num(value) def values(self): return list(self.itervalues()) # Comment below applies for the following minKey and maxKey methods # # Obscure: what if `tree` is actually empty? We're relying here on # that this class doesn't implement __delitem__: once a key gets # into an fsIndex, the only way it can go away is by invoking # clear(). Therefore nothing in _data.values() is ever empty. # # Note that because `tree` is an fsBTree, its minKey()/maxKey() methods are # very efficient. def minKey(self, key=None): if key is None: smallest_prefix = self._data.minKey() else: smallest_prefix = self._data.minKey(key[:6]) tree = self._data[smallest_prefix] assert tree if key is None: smallest_suffix = tree.minKey() else: try: smallest_suffix = tree.minKey(key[6:]) except ValueError: # 'empty tree' (no suffix >= arg) next_prefix = prefix_plus_one(smallest_prefix) smallest_prefix = self._data.minKey(next_prefix) tree = self._data[smallest_prefix] assert tree smallest_suffix = tree.minKey() return smallest_prefix + smallest_suffix def maxKey(self, key=None): if key is None: biggest_prefix = self._data.maxKey() else: biggest_prefix = self._data.maxKey(key[:6]) tree = self._data[biggest_prefix] assert tree if key is None: biggest_suffix = tree.maxKey() else: try: biggest_suffix = tree.maxKey(key[6:]) except ValueError: # 'empty tree' (no suffix <= arg) next_prefix = prefix_minus_one(biggest_prefix) biggest_prefix = self._data.maxKey(next_prefix) tree = self._data[biggest_prefix] assert tree biggest_suffix = tree.maxKey() return biggest_prefix + biggest_suffix
class MultiPointerTrack(Persistent): """ A MultiPointerTrack is a kind of structure that is based on an IOBTree, where each entry contains an ordered set (or list, depending on the implementation) of elements. Then, several "pointers" can be created, which point to different positions of the track (very much like runners in a race track). This class is abstract, implementations should be derived. """ def __init__(self, elemContainer): self._container = OOBTree() self._pointers = PersistentMapping() self._elemContainer = elemContainer # initialize first entry #self._container[timestamp(0)] = elemContainer() def addPointer(self, pid, startPos=None): """ Registers a new pointer """ self._pointers[pid] = None if startPos: self.movePointer(pid, startPos) def removePointer(self, pid): """ Removes a pointer from the list """ del self._pointers[pid] def prepareEntry(self, ts): """ Creates an empty sub-structure (elemContainer) for a given timestamp """ self._container[timestamp(ts)] = self._elemContainer() def getCurrentPosition(self, pid): """ Returns the current entry (set/list) for a given pointer id """ currentPos = self._pointers[pid] # TODO: assertion? check? return self._container[timestamp(currentPos)] def getPointerTimestamp(self, pid): """ Gets the current 'position' of a pointer (id) """ return self._pointers[pid] def __getitem__(self, ts): """ Implements __getitem__, so that mpt[timestamp] works """ if isinstance(ts, slice): return self._getSlice(ts) else: return self._container[timestamp(ts)] def _getSlice(self, s): """ Calculates a slice of the structure (timestamp-wise) """ if s.step != None: raise TypeError('Extended slices are not accepted here') return self._container.values(s.start, s.stop) def values(self, *args): """ Return values or ranges (timestamps) of the structure """ fargs = [] for a in args: if a == None: fargs.append(None) else: fargs.append(timestamp(a)) return self._container.values(*fargs) def _append(self, ts, val): """ Should be overloaded. """ raise Exception("Unimplemented method") def add(self, intTS, value): """ Adds a value to the container corresponding to a specific timestamp """ ts = timestamp(intTS) if ts not in self._container: self.prepareEntry(intTS) self._append(ts, value) def _pointerIterator(self, pid, func, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id). Takes a function that is applied to yielded values """ return self.iterate(self._pointers[pid], till, func) def iterate(self, fromPos=None, till=None, func=(lambda x: x)): """ Generator that iterates through the data structure """ if till != None: till = timestamp(till) # negative numbers mean "last but one", "last but two", etc... if till == timestamp(-1): # most common case till = self._container.maxKey() - timestamp(1) if fromPos != None: fromPos = timestamp(fromPos) for ts, entry in self._container.iteritems(till, fromPos): if fromPos and ts == fromPos: # stop immediately if we're past fromPos raise StopIteration for elem in entry: yield func((int(ts), elem)) def mostRecentTS(self, maximum=None): """ Returns most recent timestamp in track (minimum key) If 'maximum' is provided, return it if less recent """ # check that the tree has something if len(self._container) == 0: raise EmptyTrackException() mr = self._container.minKey() if maximum: maximum = timestamp(maximum) # in timestamp logic, max() returns the oldest return max(mr, maximum) else: return mr def oldestTS(self): """ Returns least recent timestamp in track (maximum key) """ # check that the tree has something if len(self._container) == 0: raise EmptyTrackException() return self._container.maxKey() def pointerIterValues(self, pid, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id) - iterates over values """ return self._pointerIterator(pid, lambda x: x[1], till=till) def pointerIterItems(self, pid, till=None): """ Iterates over the positions that are left (till the end of the track) for a given pointer (id) - iterates over key-value pairs (iteritems) """ return self._pointerIterator(pid, lambda x: x, till=till) def is_empty(self): for __, ___ in self._container.iteritems(): return False return True def movePointer(self, pid, pos): """ Moves a given pointer (id) to a given timestamp """ if pid not in self._pointers: raise KeyError("Pointer '%s' doesn't seem to exist!" % pid) # check that the tree has something if self.is_empty(): raise EmptyTrackException() self._pointers[pid] = pos def __len__(self): """ Returns the number of timestamp entries """ return len(self._container) def __delitem__(self, item): """ Deletes a given timestamp entry (or range) """ self._container.__delitem__(item) def __iter__(self): """ Iterates over the whole structure, element by elements (goes inside containers) """ return self.iterate() def __contains__(self, ts): return timestamp(ts) in self._container
class Topics(Persistent): interface.implements(ITopics) def __init__(self): self.topics = OOBTree() self.btopics = IOBTree() self.__len = Length(0) def __len__(self): return self.__len() def __iter__(self): return iter(xrange(len(self))) def __getitem__(self, key): topics = self.topics keys = topics.keys() if isinstance(key, types.SliceType): length = len(self) start = key.start or 0 stop = key.stop or length if stop > length: stop = length getitem = self.__getitem__ return [getitem(idx) for idx in range(start, stop)] else: if key < 0: key = - key - 1 else: key = len(self) - key - 1 return topics[keys[key]] def append(self, topic): self.remove(topic) id = int(topic.__name__) idx = time.time() while idx in self.topics: idx = idx + 0.00001 self.topics[idx] = topic self.btopics[id] = idx self.__len.change(1) def remove(self, topic): id = int(topic.__name__) if id in self.btopics: idx = self.btopics[id] del self.btopics[id] del self.topics[idx] self.__len.change(-1) def next(self, topic): """ return next topic """ id = int(topic.__name__) idx = self.btopics.get(id) if idx is not None: idx = idx - 0.000001 try: idx = self.topics.maxKey(idx) except ValueError: return None return self.topics.get(idx) def previous(self, topic): """ return prev topic """ id = int(topic.__name__) idx = self.btopics.get(id) if idx is not None: idx = idx + 0.000001 try: idx = self.topics.minKey(idx) except ValueError: return None return self.topics.get(idx)
class _Records(object): """The records stored in the registry. This implements dict-like access to records, where as the Registry object implements dict-like read-only access to values. """ __parent__ = None # Similar to zope.schema._field._isdotted, but allows up to one '/' _validkey = re.compile(r"([a-zA-Z][a-zA-Z0-9_-]*)" r"([.][a-zA-Z][a-zA-Z0-9_-]*)*" r"([/][a-zA-Z][a-zA-Z0-9_-]*)?" r"([.][a-zA-Z][a-zA-Z0-9_-]*)*" # use the whole line r"$").match def __init__(self, parent): self.__parent__ = parent self._fields = OOBTree() self._values = OOBTree() def __setitem__(self, name, record): if not self._validkey(name): raise InvalidRegistryKey(record) if not IRecord.providedBy(record): raise ValueError("Value must be a record") self._setField(name, record.field) self._values[name] = record.value record.__name__ = name record.__parent__ = self.__parent__ notify(RecordAddedEvent(record)) def __delitem__(self, name): record = self[name] # unbind the record so that it won't attempt to look up values from # the registry anymore record.__parent__ = None del self._fields[name] del self._values[name] notify(RecordRemovedEvent(record)) def __getitem__(self, name): field = self._getField(name) value = self._values[name] record = Record(field, value, _validate=False) record.__name__ = name record.__parent__ = self.__parent__ return record def get(self, name, default=None): try: return self[name] except KeyError: return default def __nonzero__(self): return self._values.__nonzero__() def __len__(self): return self._values.__len__() def __iter__(self): return self._values.__iter__() def has_key(self, name): return self._values.__contains__(name) def __contains__(self, name): return self._values.__contains__(name) def keys(self, min=None, max=None): return self._values.keys(min, max) def maxKey(self, key=None): return self._values.maxKey(key) def minKey(self, key=None): return self._values.minKey(key) def values(self, min=None, max=None): return [self[name] for name in self.keys(min, max)] def items(self, min=None, max=None): return [( name, self[name], ) for name in self.keys(min, max)] def setdefault(self, key, value): if key not in self: self[key] = value return self[key] def clear(self): self._fields.clear() self._values.clear() # Helper methods def _getField(self, name): field = self._fields[name] # Handle field reference pointers if isinstance(field, basestring): recordName = field while isinstance(field, basestring): recordName = field field = self._fields[recordName] field = FieldRef(recordName, field) return field def _setField(self, name, field): if not IPersistentField.providedBy(field): raise ValueError("The record's field must be an IPersistentField.") if IFieldRef.providedBy(field): if field.recordName not in self._fields: raise ValueError( "Field reference points to non-existent record") self._fields[name] = field.recordName # a pointer, of sorts else: field.__name__ = 'value' self._fields[name] = field
class DataBucketStream(Document): """ Represents data stored in many small files inside a "stream". Each file is "addressed" by its key similar to dict. """ meta_type = 'ERP5 Data Bucket Stream' portal_type = 'Data Bucket Stream' add_permission = Permissions.AddPortalContent # Declarative security security = ClassSecurityInfo() security.declareObjectProtected(Permissions.AccessContentsInformation) # Declarative properties property_sheets = (PropertySheet.CategoryCore, PropertySheet.SortIndex) def __init__(self, id, **kw): self.initBucketTree() self.initIndexTree() Document.__init__(self, id, **kw) def __len__(self): return len(self._tree) def initBucketTree(self): """ Initialize the Bucket Tree """ self._tree = OOBTree() def initIndexTree(self): """ Initialize the Index Tree """ self._long_index_tree = LOBTree() def getMaxKey(self, key=None): """ Return the maximum key """ try: return self._tree.maxKey(key) except ValueError: return None def getMaxIndex(self, index=None): """ Return the maximum index """ try: return self._long_index_tree.maxKey(index) except ValueError: return None def getMinKey(self, key=None): """ Return the minimum key """ try: return self._tree.minKey(key) except ValueError: return None def getMinIndex(self, index=None): """ Return the minimum key """ try: return self._long_index_tree.minKey(index) except ValueError: return None def _getOb(self, id, *args, **kw): return None def getBucketByKey(self, key=None): """ Get one bucket """ return self._tree[key].value def getBucketByIndex(self, index=None): """ Get one bucket """ key = self._long_index_tree[index] return self.getBucketByKey(key).value def getBucket(self, key): log('DeprecationWarning: Please use getBucketByKey') return self.getBucketByKey(key) def hasBucketKey(self, key): """ Wether bucket with such key exists """ return key in self._tree def hasBucketIndex(self, index): """ Wether bucket with such index exists """ return self._long_index_tree.has_key(index) def insertBucket(self, key, value): """ Insert one bucket """ try: count = self._long_index_tree.maxKey() + 1 except ValueError: count = 0 except AttributeError: pass try: self._long_index_tree.insert(count, key) except AttributeError: pass value = PersistentString(value) is_new_key = self._tree.insert(key, value) if not is_new_key: self.log("Reingestion of same key") self._tree[key] = value def getBucketKeySequenceByKey(self, start_key=None, stop_key=None, count=None, exclude_start_key=False, exclude_stop_key=False): """ Get a lazy sequence of bucket keys """ sequence = self._tree.keys(min=start_key, max=stop_key, excludemin=exclude_start_key, excludemax=exclude_stop_key) if count is None: return sequence return sequence[:count] def getBucketKeySequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket keys """ sequence = self._long_index_tree.values(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is None: return sequence return sequence[:count] def getBucketKeySequence(self, start_key=None, count=None): log('DeprecationWarning: Please use getBucketKeySequenceByKey') return self.getBucketKeySequenceByKey(start_key=start_key, count=count) def getBucketIndexKeySequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket keys """ sequence = self._long_index_tree.items(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is not None: sequence = sequence[:count] return IndexKeySequence(self, sequence) def getBucketIndexSequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket keys """ sequence = self._long_index_tree.keys(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is None: return sequence return sequence[:count] def getBucketValueSequenceByKey(self, start_key=None, stop_key=None, count=None, exclude_start_key=False, exclude_stop_key=False): """ Get a lazy sequence of bucket values """ sequence = self._tree.values(min=start_key, max=stop_key, excludemin=exclude_start_key, excludemax=exclude_stop_key) if count is None: return sequence return sequence[:count] def getBucketValueSequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket values """ sequence = self._long_index_tree.values(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is not None: sequence = sequence[:count] return IndexValueSequence(self, sequence) def getBucketValueSequence(self, start_key=None, count=None): log('DeprecationWarning: Please use getBucketValueSequenceByKey') return self.getBucketValueSequenceByKey(start_key=start_key, count=count) def getBucketKeyItemSequenceByKey(self, start_key=None, stop_key=None, count=None, exclude_start_key=False, exclude_stop_key=False): """ Get a lazy sequence of bucket items """ sequence = self._tree.items(min=start_key, max=stop_key, excludemin=exclude_start_key, excludemax=exclude_stop_key) if count is None: return sequence return sequence[:count] def getBucketItemSequence(self, start_key=None, count=None, exclude_start_key=False): log('DeprecationWarning: Please use getBucketKeyItemSequenceByKey') return self.getBucketKeyItemSequenceByKey( start_key=start_key, count=count, exclude_start_key=exclude_start_key) def getBucketIndexItemSequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket items """ sequence = self._long_index_tree.items(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is not None: sequence = sequence[:count] return IndexItemSequence(self, sequence) def getBucketIndexKeyItemSequenceByIndex(self, start_index=None, stop_index=None, count=None, exclude_start_index=False, exclude_stop_index=False): """ Get a lazy sequence of bucket items """ sequence = self._long_index_tree.items(min=start_index, max=stop_index, excludemin=exclude_start_index, excludemax=exclude_stop_index) if count is not None: sequence = sequence[:count] return IndexKeyItemSequence(self, sequence) def getItemList(self): """ Return a list of all key, value pairs """ return [item for item in self._tree.items()] def getKeyList(self): """ Return a list of all keys """ return [key for key in self._tree.keys()] def getIndexList(self): """ Return a list of all indexes """ return [key for key in self._long_index_tree.keys()] def getIndexKeyTupleList(self): """ Return a list of all indexes """ return [key for key in self._long_index_tree.items()] def getMd5sum(self, key): """ Get hexdigest of bucket. """ h = hashlib.md5() h.update(self.getBucketByKey(key)) return h.hexdigest() def delBucketByKey(self, key): """ Remove the bucket. """ del self._tree[key] for index, my_key in list(self.getBucketIndexKeySequenceByIndex()): if my_key == key: del self._long_index_tree[index] def delBucketByIndex(self, index): """ Remove the bucket. """ key = self._long_index_tree[index] del self._tree[key] del self._long_index_tree[index] def rebuildIndexTreeByKeyOrder(self): """ Clear and rebuild the index tree by order of keys """ self.initIndexTree() for count, key in enumerate(self.getBucketKeySequenceByKey()): self._long_index_tree.insert(count, key)
class SingleValueIndex(Index): """An `index <Index>` where each key may only point to a single value.""" accepts_multiple_values = False @overrides(Index.__init__) def __init__(self, pairs = None): self.__items = OOBTree() self.__descending_items = OOBTree() Index.__init__(self, pairs) @overrides(Index.add) def add(self, key, value): if key is None: raise ValueError("Can't use None as a key for a SingleValueIndex") self.__items[key] = value self.__descending_items[Descending(key)] = value @overrides(Index.remove) def remove(self, key, value = undefined): if value is not undefined \ and self.get(key, undefined) != value: return if value is undefined: try: del self.__items[key] del self.__descending_items[key] except KeyError: pass @overrides(Index.items) def items( self, min = undefined, max = undefined, exclude_min = False, exclude_max = False, descending = False ): min = self.__boundary(min, descending) max = self.__boundary(max, descending) if descending: min, max = max, min exclude_min, exclude_max = exclude_max, exclude_min for desc_key, value in self.__descending_items.iteritems( min = min, max = max, excludemin = exclude_min, excludemax = exclude_max ): yield (desc_key.value, value) else: for pair in self.__items.iteritems( min = min, max = max, excludemin = exclude_min, excludemax = exclude_max ): yield pair def __boundary(self, boundary, descending): if boundary is undefined: return None return Descending(boundary) if descending else boundary @overrides(Index.min_key) def min_key(self, exclude_none = False): if exclude_none: for key in list(self.keys()): if key is not None: return key else: return self.__items.minKey() @overrides(Index.max_key) def max_key(self): return self.__descending_items.minKey().value @overrides(Index.__len__) def __len__(self): return len(self.__items) @overrides(Index.__bool__) def __bool__(self): return bool(self.__items) @overrides(Index.__contains__) def __contains__(self, key): return key in self.__items def __getitem__(self, key): """Get the value for the specified key. :param key: The key to retrieve the value for. :return: The value for the specified key. :raise KeyError: Raised if the indicated key isn't present in the index. """ if isinstance(key, slice): raise ValueError( "Slicing an index is not supported; use keys()/values() " "instead") else: return self.__items[key] def get(self, key, default = None): """Get the value for the specified key, returning `default` if the key is undefined. :param key: The key to retrieve the value for. :param default: The value that should be returned if the key is not defined by the index. :return: The value for the specified key. """ return self.__items.get(key, default)
class _Records(object): """The records stored in the registry. This implements dict-like access to records, where as the Registry object implements dict-like read-only access to values. """ __parent__ = None # Similar to zope.schema._field._isdotted, but allows up to one '/' _validkey = re.compile( r"([a-zA-Z][a-zA-Z0-9_-]*)" r"([.][a-zA-Z][a-zA-Z0-9_-]*)*" r"([/][a-zA-Z][a-zA-Z0-9_-]*)?" r"([.][a-zA-Z][a-zA-Z0-9_-]*)*" # use the whole line r"$").match def __init__(self, parent): self.__parent__ = parent self._fields = OOBTree() self._values = OOBTree() def __setitem__(self, name, record): if not self._validkey(name): raise InvalidRegistryKey(record) if not IRecord.providedBy(record): raise ValueError("Value must be a record") self._setField(name, record.field) self._values[name] = record.value record.__name__ = name record.__parent__ = self.__parent__ notify(RecordAddedEvent(record)) def __delitem__(self, name): record = self[name] # unbind the record so that it won't attempt to look up values from # the registry anymore record.__parent__ = None del self._fields[name] del self._values[name] notify(RecordRemovedEvent(record)) def __getitem__(self, name): field = self._getField(name) value = self._values[name] record = Record(field, value, _validate=False) record.__name__ = name record.__parent__ = self.__parent__ return record def get(self, name, default=None): try: return self[name] except KeyError: return default def __nonzero__(self): return self._values.__nonzero__() def __len__(self): return self._values.__len__() def __iter__(self): return self._values.__iter__() def has_key(self, name): return self._values.__contains__(name) def __contains__(self, name): return self._values.__contains__(name) def keys(self, min=None, max=None): return self._values.keys(min, max) def maxKey(self, key=None): return self._values.maxKey(key) def minKey(self, key=None): return self._values.minKey(key) def values(self, min=None, max=None): return [self[name] for name in self.keys(min, max)] def items(self, min=None, max=None): return [(name, self[name],) for name in self.keys(min, max)] def setdefault(self, key, value): if key not in self: self[key] = value return self[key] def clear(self): self._fields.clear() self._values.clear() # Helper methods def _getField(self, name): field = self._fields[name] # Handle field reference pointers if isinstance(field, basestring): recordName = field while isinstance(field, basestring): recordName = field field = self._fields[recordName] field = FieldRef(recordName, field) return field def _setField(self, name, field): if not IPersistentField.providedBy(field): raise ValueError("The record's field must be an IPersistentField.") if IFieldRef.providedBy(field): if field.recordName not in self._fields: raise ValueError( "Field reference points to non-existent record" ) self._fields[name] = field.recordName # a pointer, of sorts else: field.__name__ = 'value' self._fields[name] = field
class EventIndex(SimpleItem): implements(IPluggableIndex) meta_type = "EventIndex" manage_options = ( {'label': 'Settings', 'action': 'manage_main'}, ) manage = manage_main = DTMLFile('www/manageEventIndex', globals()) manage_main._setName('manage_main') def __init__(self, id, extra=None, caller=None): self._id = id self.start_attr = extra and extra['start_attr'] or 'start' self.end_attr = extra and extra['end_attr'] or 'end' self.recurrence_attr = extra and extra['recurrence_attr'] or 'recurrence' self.clear() def clear(self): """Empty the index""" self._length = Length() self._end2uid = OOBTree() self._start2uid = OOBTree() self._uid2end = IOBTree() # Contains the index used in _end2uid self._uid2duration = IOBTree() # Contains the duration self._uid2start = IOBTree() self._uid2recurrence = IOBTree() def getId(self): """Return Id of index.""" return self._id def getEntryForObject(self, documentId, default=''): """Get all information contained for 'documentId'.""" uid2start = self._uid2start.get(documentId) if uid2start: return { 'start': uid2start, 'end': self._uid2end[documentId], 'recurrence': self._uid2recurrence[documentId], 'duration': self._uid2duration[documentId] } else: return default def getIndexSourceNames(self): """Get a sequence of attribute names that are indexed by the index. """ return self.start_attr, self.end_attr, self.recurrence_attr def _getattr(self, name, obj): attr = getattr(obj, name, None) if callable(attr): attr = attr() if isinstance(attr, DateTime): attr = attr.utcdatetime() return attr def index_object(self, documentId, obj, threshold=None): """Index an object. - ``documentId`` is the integer ID of the document. - ``obj`` is the object to be indexed. - ``threshold`` is the number of words to process between committing subtransactions. If None, subtransactions are disabled. For each name in ``getIndexSourceNames``, try to get the named attribute from ``obj``. - If the object does not have the attribute, do not add it to the index for that name. - If the attribute is a callable, call it to get the value. If calling it raises an AttributeError, do not add it to the index. for that name. """ # Clear the data structures before indexing the object. This will ensure # we don't leave any stale data behind when an object gets reindexed. self.unindex_object(documentId) ### 1. Get the values. start = self._getattr(self.start_attr, obj) end = self._getattr(self.end_attr, obj) if start is None: # Ignore calls if the obj does not have the start field. return False if end is None: # Singular event end = start recurrence = self._getattr(self.recurrence_attr, obj) if not recurrence: rule = None elif isinstance(recurrence, basestring): # XXX trap and log errors rule = rrule.rrulestr(recurrence, dtstart=start) elif isinstance(recurrence, rrule.rrulebase): rule = recurrence else: #XXX Log error rule = None # Strip out times from the recurrence: if rule is not None: sync_timezone(rule, start.tzinfo) ### 2. Make them into what should be indexed. # XXX Naive events are not comparable to timezoned events, so we convert # everything to utctimetuple(). This means naive events are assumed to # be GMT, but we can live with that at the moment. start_value = start.utctimetuple() end_value = end.utctimetuple() # The end value should be the end of the recurrence, if any: if rule is not None: if is_open_ended(rule): # This recurrence is open ended end_value = None else: duration = end - start allrecs = [x for x in rule._iter()] if allrecs: last = allrecs[-1] + duration else: # Real data may have invalud recurrence rules, # which end before the start for example. # Then we end up here. last = end end_value = last.utctimetuple() ### 3. Store everything in the indexes: row = self._start2uid.get(start_value, None) if row is None: row = IITreeSet((documentId,)) self._start2uid[start_value] = row else: row.insert(documentId) row = self._end2uid.get(end_value, None) if row is None: row = IITreeSet((documentId,)) self._end2uid[end_value] = row else: row.insert(documentId) self._uid2start[documentId] = start_value self._uid2recurrence[documentId] = rule self._uid2end[documentId] = end_value self._uid2duration[documentId] = end - start return True def _remove_id(self, documentId, from_uid, to_uid): """Remove documentId based on point. Helper method for unindex_object method. :param documentId: Integer :type documentId: int :param from_uid: :type to_uid: :param to_uid: :type to_uid: """ fuid = from_uid.pop(documentId, 'No ID found') row = to_uid.get(fuid) if row is not None: if documentId in row: row.remove(documentId) if len(row) == 0: to_uid.pop(fuid, 'Not Found') def unindex_object(self, documentId): """Remove the documentId from the index.""" self._remove_id(documentId, self._uid2start, self._start2uid) self._remove_id(documentId, self._uid2end, self._end2uid) self._uid2duration.pop(documentId, 'No ID found') self._uid2recurrence.pop(documentId, 'No ID found') def _get_position(self, request, position): """Get position from certain ID. :param request: Request :type request: object :param position: start or end :type position: str """ pos = request[self._id].get(position) if isinstance(pos, DateTime): pos = pos.utcdatetime() return pos def _finalize_index(self, result, start, end, used_fields): filtered_result = IITreeSet() # used_recurrence = False for documentId in result: recurrence = self._uid2recurrence.get(documentId) if recurrence is None: # This event isn't recurring, so it's a match: filtered_result.add(documentId) continue # used_recurrence = True match = False # This is a possible place where optimizations can be done if # necessary. For example, for periods where the start and end # date is the same, we can first check if the start time and # and time of the date falls inbetween the start and end times # of the period, so to avoid expansion. But most likely this # will have a very small impact on speed, so I skip this until # it actually becomes a problem. if start is not None: event_start = datetime(*self._uid2start[documentId][:6]) else: event_start = None if end is not None: event_duration = self._uid2duration[documentId] event_end = event_start + event_duration else: event_end = None for occurrence in recurrence._iter(): utc_occurrence = datetime(*occurrence.utctimetuple()[:6]) if event_start is not None and utc_occurrence < event_start: # XXX we should add a counter and break after 10000 occurrences. continue if event_end is not None and utc_occurrence > event_end: break # The start of this occurrence starts between the start and end date of # the query: match = True break if match: filtered_result.add(documentId) # if used_recurrence: used_fields += (self.recurrence_attr,) return filtered_result, used_fields def _apply_index(self, request, resultset=None): """Apply the index to query parameters given in 'request'. The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column and this parameter is either a Record or a class instance then it is assumed that the parameters of this index are passed as attribute (Note: this is the recommended way to pass parameters since Zope 2.4) Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. The resultset argument contains the resultset, as already calculated by ZCatalog's search method. """ if not request.has_key(self._id): # 'in' doesn't work with this object return IITreeSet(self._uid2end.keys()), () start = self._get_position(request, 'start') end = self._get_position(request, 'end') used_fields = () # We don't want the events who end before the start. In other # words we want to find those evens whose end >= the start query, # or None as None means they have infinite recurrence. try: maxkey = self._end2uid.maxKey() except ValueError: # No events at all return IITreeSet(), used_fields if start is None or maxkey is None: # start is None, so we need to search right from the start; or # (amazingly) all events have infinite recurrence. # This means we must return *all* uids. start_uids = IITreeSet(self._uid2end.keys()) else: used_fields += (self.start_attr,) #start_uids = IITreeSet() start = start.utctimetuple() try: minkey = self._end2uid.minKey(start) # Events that end on exactly the same same time as the # search period start should not be included: if minkey == start: excludemin = True else: excludemin = False start_uids = multiunion(self._end2uid.values(minkey, maxkey, excludemin=excludemin)) except ValueError: # No ending events start_uids = IITreeSet() # Include open ended events, if any if self._end2uid.has_key(None): start_uids = union(start_uids, self._end2uid[None]) # XXX At this point an intersection with the resultset might be # beneficial. It would stop us from calculating the recurrence # of ids that won't be returned. It could be done after the # intersection with end_uids below as well, performance tests will tell. # We also do not want the events whose start come after the end query. # In other words, we find all events where start <= end. if end is not None: end = end.utctimetuple() try: minkey = self._start2uid.minKey() end_uids = multiunion(self._start2uid.values(minkey, end)) used_fields += (self.end_attr,) except ValueError: # No events return IITreeSet(), used_fields result = intersection(start_uids, end_uids) else: # No end specified, take all: result = start_uids return self._finalize_index(result, start, end, used_fields) def numObjects(self): """Return the number of indexed objects.""" return len(self._uid2start.keys())
class fsIndex(object): def __init__(self): self._data = OOBTree() def __getitem__(self, key): return str2num(self._data[key[:6]][key[6:]]) def get(self, key, default=None): tree = self._data.get(key[:6], default) if tree is default: return default v = tree.get(key[6:], default) if v is default: return default return str2num(v) def __setitem__(self, key, value): value = num2str(value) treekey = key[:6] tree = self._data.get(treekey) if tree is None: tree = fsBucket() self._data[treekey] = tree tree[key[6:]] = value def __delitem__(self, key): treekey = key[:6] tree = self._data.get(treekey) if tree is None: raise KeyError, key del tree[key[6:]] if not tree: del self._data[treekey] def __len__(self): r = 0 for tree in self._data.itervalues(): r += len(tree) return r def update(self, mapping): for k, v in mapping.items(): self[k] = v def has_key(self, key): v = self.get(key, self) return v is not self def __contains__(self, key): tree = self._data.get(key[:6]) if tree is None: return False v = tree.get(key[6:], None) if v is None: return False return True def clear(self): self._data.clear() def __iter__(self): for prefix, tree in self._data.iteritems(): for suffix in tree: yield prefix + suffix iterkeys = __iter__ def keys(self): return list(self.iterkeys()) def iteritems(self): for prefix, tree in self._data.iteritems(): for suffix, value in tree.iteritems(): yield (prefix + suffix, str2num(value)) def items(self): return list(self.iteritems()) def itervalues(self): for tree in self._data.itervalues(): for value in tree.itervalues(): yield str2num(value) def values(self): return list(self.itervalues()) # Comment below applies for the following minKey and maxKey methods # # Obscure: what if `tree` is actually empty? We're relying here on # that this class doesn't implement __delitem__: once a key gets # into an fsIndex, the only way it can go away is by invoking # clear(). Therefore nothing in _data.values() is ever empty. # # Note that because `tree` is an fsBTree, its minKey()/maxKey() methods are # very efficient. def minKey(self, key=None): if key is None: smallest_prefix = self._data.minKey() else: smallest_prefix = self._data.minKey(key[:6]) tree = self._data[smallest_prefix] assert tree if key is None: smallest_suffix = tree.minKey() else: try: smallest_suffix = tree.minKey(key[6:]) except ValueError: # 'empty tree' (no suffix >= arg) next_prefix = prefix_plus_one(smallest_prefix) smallest_prefix = self._data.minKey(next_prefix) tree = self._data[smallest_prefix] assert tree smallest_suffix = tree.minKey() return smallest_prefix + smallest_suffix def maxKey(self, key=None): if key is None: biggest_prefix = self._data.maxKey() else: biggest_prefix = self._data.maxKey(key[:6]) tree = self._data[biggest_prefix] assert tree if key is None: biggest_suffix = tree.maxKey() else: try: biggest_suffix = tree.maxKey(key[6:]) except ValueError: # 'empty tree' (no suffix <= arg) next_prefix = prefix_minus_one(biggest_prefix) biggest_prefix = self._data.maxKey(next_prefix) tree = self._data[biggest_prefix] assert tree biggest_suffix = tree.maxKey() return biggest_prefix + biggest_suffix
class fsIndex(object): def __init__(self): self._data = OOBTree() def __getitem__(self, key): return str2num(self._data[key[:6]][key[6:]]) def get(self, key, default=None): tree = self._data.get(key[:6], default) if tree is default: return default v = tree.get(key[6:], default) if v is default: return default return str2num(v) def __setitem__(self, key, value): value = num2str(value) treekey = key[:6] tree = self._data.get(treekey) if tree is None: tree = fsBucket() self._data[treekey] = tree tree[key[6:]] = value def __len__(self): r = 0 for tree in self._data.itervalues(): r += len(tree) return r def update(self, mapping): for k, v in mapping.items(): self[k] = v def has_key(self, key): v = self.get(key, self) return v is not self def __contains__(self, key): tree = self._data.get(key[:6]) if tree is None: return False v = tree.get(key[6:], None) if v is None: return False return True def clear(self): self._data.clear() def __iter__(self): for prefix, tree in self._data.iteritems(): for suffix in tree: yield prefix + suffix iterkeys = __iter__ def keys(self): return list(self.iterkeys()) def iteritems(self): for prefix, tree in self._data.iteritems(): for suffix, value in tree.iteritems(): yield (prefix + suffix, str2num(value)) def items(self): return list(self.iteritems()) def itervalues(self): for tree in self._data.itervalues(): for value in tree.itervalues(): yield str2num(value) def values(self): return list(self.itervalues()) # Comment below applies for the following minKey and maxKey methods # # Obscure: what if `tree` is actually empty? We're relying here on # that this class doesn't implement __delitem__: once a key gets # into an fsIndex, the only way it can go away is by invoking # clear(). Therefore nothing in _data.values() is ever empty. # # Note that because `tree` is an fsBTree, its minKey()/maxKey() methods are # very efficient. def minKey(self, key=None): if key is None: smallest_prefix = self._data.minKey() else: smallest_prefix = self._data.minKey(key[:6]) tree = self._data[smallest_prefix] assert tree if key is None: smallest_suffix = tree.minKey() else: try: smallest_suffix = tree.minKey(key[6:]) except ValueError: # 'empty tree' (no suffix >= arg) next_prefix = prefix_plus_one(smallest_prefix) smallest_prefix = self._data.minKey(next_prefix) tree = self._data[smallest_prefix] assert tree smallest_suffix = tree.minKey() return smallest_prefix + smallest_suffix def maxKey(self, key=None): if key is None: biggest_prefix = self._data.maxKey() else: biggest_prefix = self._data.maxKey(key[:6]) tree = self._data[biggest_prefix] assert tree if key is None: biggest_suffix = tree.maxKey() else: try: biggest_suffix = tree.maxKey(key[6:]) except ValueError: # 'empty tree' (no suffix <= arg) next_prefix = prefix_minus_one(biggest_prefix) biggest_prefix = self._data.maxKey(next_prefix) tree = self._data[biggest_prefix] assert tree biggest_suffix = tree.maxKey() return biggest_prefix + biggest_suffix