def __init__(self): # These keep track of symbolic label and branch names that # have been used to ensure that they don't collide. self._branches = OIBTree() self._branches['mainline'] = 1 self._labels = OIBTree() self._histories = OOBTree() self._created = time.time()
def setupAnnotations(context): """ set up the annotations if they haven't been set up already. The rest of the functions in here assume that this has already been set up """ annotations = IAnnotations(context) if yays not in annotations: annotations[yays] = OIBTree() if nays not in annotations: annotations[nays] = OIBTree() return annotations
def storage(self): """ get the counter storage """ annotation = get_portal_annotation() if annotation.get(NUMBER_STORAGE) is None: annotation[NUMBER_STORAGE] = OIBTree() return annotation[NUMBER_STORAGE]
def clear(self): """Empty the lexicon. """ self.length = Length() self._wid_length_based = False self._wids = OIBTree() # word -> wid self._words = IOBTree() # wid -> word
def resetOrdering(ordering): annotations = IAnnotations(ordering.context) order = PersistentList() annotations.__setitem__(ordering.ORDER_KEY, order) pos = OIBTree() annotations.__setitem__(ordering.POS_KEY, pos) return order, pos
def clear(self): self.data = IOBTree() # {rid -> record as tuple} self.indexes = {} # {index_name -> OOBTree({value -> IITreeSet})} self.primary_index = OIBTree() # {primary key -> rid} for position, column in self.col_info: if column.indexed: self.indexes[column.name] = OOBTree()
def convert_to_uuidindex(catalog, index): if isinstance(index, UUIDIndex): return logger.info('Converting index `%s` to UUIDIndex.' % index.getId()) index.__class__ = UUIDIndex index._p_changed = True catalog._catalog._p_changed = True # convert from OOBTree to OIBTree old_index = index._index if not isinstance(old_index, OIBTree): index._index = _index = OIBTree() for k, v in old_index.items(): if k is None: continue if isinstance(v, int): _index[k] = v else: if isinstance(v, (IISet, IITreeSet)): # inconsistent data, one uid with multiple docids paths = dict((tuple(catalog.getpath(k).split('/')), k) for k in v.keys()) shortest = min(paths, key=len) for path, key in paths.iteritems(): if path[:len(shortest)] != shortest: raise ValueError( 'Inconsistent UID index, UID %s is associated ' 'with multiple docids: %r' % (k, paths)) # All other docids are sub-paths of another # indicating the UID was just acquired, # choose the shortest _index[k] = paths[shortest] del old_index transaction.savepoint(optimistic=True) logger.info('Finished conversion.')
def clear(self): self._length = Length() self._index = OIBTree() self._unindex = IOBTree() if self._counter is None: self._counter = Length() else: self._increment_counter()
def _cleanup(self): """Cleans up errors in the BTrees. Certain ZODB bugs have caused BTrees to become slightly insane. Fortunately, there is a way to clean up damaged BTrees that always seems to work: make a new BTree containing the items() of the old one. Returns 1 if no damage was detected, or 0 if damage was detected and fixed. """ from BTrees.check import check path = '/'.join(self.getPhysicalPath()) try: check(self._tree) for key in self._tree.keys(): if key not in self._tree: raise AssertionError( "Missing value for key: %s" % repr(key)) check(self._mt_index) keys = set(self._tree.keys()) for key, value in self._mt_index.items(): if (key not in self._mt_index or self._mt_index[key] is not value): raise AssertionError( "Missing or incorrect meta_type index: %s" % repr(key)) check(value) for k in value.keys(): if k not in value or k not in keys: raise AssertionError( "Missing values for meta_type index: %s" % repr(key)) return 1 except AssertionError: LOG.warn('Detected damage to %s. Fixing now.' % path, exc_info=sys.exc_info()) try: self._tree = OOBTree(self._tree) keys = set(self._tree.keys()) mt_index = OOBTree() for key, value in self._mt_index.items(): for name in tuple(value.keys()): if name not in keys: del value[name] mt_index[key] = OIBTree(value) self._mt_index = mt_index new = len(keys) if self._count() != new: self._count.set(new) except: LOG.error('Failed to fix %s.' % path, exc_info=sys.exc_info()) raise else: LOG.info('Fixed %s.' % path) return 0
def initializeAnnotations(obj, event): """Ensure that we don't delegate certain annotations by setting them from the beginning. """ annotations = IAnnotations(obj) annotations.setdefault(DefaultOrdering.ORDER_KEY, PersistentList()) annotations.setdefault(DefaultOrdering.POS_KEY, OIBTree()) annotations.setdefault(CONTENTRULES_KEY, None) annotations.setdefault(CONTEXT_ASSIGNMENT_KEY, OOBTree())
def noteLock(self, obj, user_id): mapping = getattr(self, '_locks', None) if mapping is None: mapping = self._locks = OOBTree() path = '/'.join(obj.getPhysicalPath()) items = mapping.get(user_id, None) if items is None: items = OIBTree() mapping[user_id] = items items[path] = 1
def __init__(self, *pipeline): self._wids = OIBTree() # word -> wid self._words = IOBTree() # wid -> word # wid 0 is reserved for words that aren't in the lexicon (OOV -- out # of vocabulary). This can happen, e.g., if a query contains a word # we never saw before, and that isn't a known stopword (or otherwise # filtered out). Returning a special wid value for OOV words is a # way to let clients know when an OOV word appears. self.wordCount = Length() self._pipeline = pipeline
def clear(self): """ clear catalog """ self.data = IOBTree() # mapping of rid to meta_data self.uids = OIBTree() # mapping of uid to rid self.paths = IOBTree() # mapping of rid to uid self._length = BTrees.Length.Length() for index in self.indexes.keys(): self.getIndex(index).clear()
def load_model(): data = DataFS() keys, label = data.gen_train_data() btree = OIBTree() rubbish_k = np.load('6k.npy') rubbish_v = np.load('6v.npy') w = np.load('w.npy', allow_pickle=True) b = np.load('b.npy', allow_pickle=True) for i in range(len(rubbish_k)): btree[str(rubbish_k[i])] = int(rubbish_v[i]) return keys, label, w, b, btree
def setupAnnotations(context): """ set up the annotations if they haven't been set up already. The rest of the functions in here assume that this has already been set up """ annotations = IAnnotations(context) changed = False if yays not in annotations: annotations[yays] = OIBTree() changed = True if nays not in annotations: annotations[nays] = OIBTree() changed = True if changed: request = getRequest() alsoProvides(request, IDisableCSRFProtection) return annotations
def __init__(self, id="++conversation++default"): self.id = id # username -> count of comments; key is removed when count reaches 0 self._commentators = OIBTree() # id -> comment - find comment by id self._comments = LOBTree() # id -> LLSet (children) - find all children for a given comment. # 0 signifies root. self._children = LOBTree()
def importNode(self, node, mode=PURGE): """Import the object from the DOM node. """ pipeline = [] for child in node.childNodes: if child.nodeName == 'element': element = element_factory.instantiate( child.getAttribute('group'), child.getAttribute('name')) pipeline.append(element) self.context._pipeline = tuple(pipeline) #clear lexicon self.context._wids = OIBTree() self.context._words = IOBTree() self.context.length = Length()
def _setOb(self, id, object): """Store the named object in the folder. """ tree = self._tree if tree.has_key(id): raise KeyError('There is already an item named "%s".' % id) tree[id] = object self._count.change(1) # Update the meta type index. mti = self._mt_index meta_type = getattr(object, 'meta_type', None) if meta_type is not None: ids = mti.get(meta_type, None) if ids is None: ids = OIBTree() mti[meta_type] = ids ids[id] = 1
def create_token(self, userid, timeout=None, data=None): payload = {} payload['sub'] = userid if timeout is None: timeout = self.token_timeout if timeout: payload['exp'] = datetime.utcnow() + timedelta(seconds=timeout) if data is not None: payload.update(data) token = jwt.encode(payload, self._signing_secret(), algorithm='HS256') if self.store_tokens: if self._tokens is None: self._tokens = OOBTree() if userid not in self._tokens: self._tokens[userid] = OIBTree() self._tokens[userid][token] = int(time.time()) return token
def _getCacheId(self): """Return a cache id for preferences. We use: - user_id: because preferences are always different by user - self._preference_cache[user_id] which is increased everytime a user preference is modified - self._preference_cache[None] which is increased everytime a global preference is modified """ user_id = getSecurityManager().getUser().getId() try: self._preference_cache except AttributeError: self._preference_cache = OIBTree() return self._preference_cache.get(None), self._preference_cache.get( user_id), user_id
def __init__(self, id=None): super(LinkCheckTool, self).__init__(id) # This is the work queue; items in this queue are scheduled # for link validity check. self.queue = CompositeQueue() # Additional queue for internal crawler to revalidate the site self.crawl_queue = CompositeQueue() # This is the link database. It maps a hyperlink index to a # tuple (timestamp, status, referers). self.checked = IOBTree() # Indexes self.index = OIBTree() self.links = IOBTree() # This is a counter that allows us to add new hyperlinks and # provide an indexc quickly. self.counter = 0
def testCleanup(self): self.assert_(self.f._cleanup()) key = TrojanKey('a') self.f._tree[key] = 'b' self.assert_(self.f._cleanup()) key.value = 'z' # With a key in the wrong place, there should now be damage. self.assert_(not self.f._cleanup()) # Now it's fixed. self.assert_(self.f._cleanup()) from BTrees.OIBTree import OIBTree tree = self.f._mt_index['d'] = OIBTree() tree['e'] = 1 self.assert_(not self.f._cleanup()) # Verify the management interface also works, # but don't test return values. self.f.manage_cleanup() key.value = 'a' self.f.manage_cleanup()
def _convertBTrees(self, threshold=200): if (type(self._lexicon) is OIBTree and type(getattr(self, '_inverseLex', None)) is IOBTree): return from BTrees.convert import convert lexicon=self._lexicon self._lexicon=OIBTree() self._lexicon._p_jar=self._p_jar convert(lexicon, self._lexicon, threshold) try: inverseLex=self._inverseLex self._inverseLex=IOBTree() except AttributeError: # older lexicons didn't have an inverse lexicon self._inverseLex=IOBTree() inverseLex=self._inverseLex self._inverseLex._p_jar=self._p_jar convert(inverseLex, self._inverseLex, threshold)
def __init__(self, datafs, writable=0, trans=0, pack=0): self.trans_limit = trans self.pack_limit = pack self.trans_count = 0 self.pack_count = 0 self.stopdict = get_stopdict() self.mh = mhlib.MH() self.filestorage = FileStorage(datafs, read_only=(not writable)) self.database = DB(self.filestorage) self.connection = self.database.open() self.root = self.connection.root() try: self.index = self.root["index"] except KeyError: self.index = self.root["index"] = TextIndex() try: self.docpaths = self.root["docpaths"] except KeyError: self.docpaths = self.root["docpaths"] = IOBTree() try: self.doctimes = self.root["doctimes"] except KeyError: self.doctimes = self.root["doctimes"] = IIBTree() try: self.watchfolders = self.root["watchfolders"] except KeyError: self.watchfolders = self.root["watchfolders"] = {} self.path2docid = OIBTree() for docid in self.docpaths.keys(): path = self.docpaths[docid] self.path2docid[path] = docid try: self.maxdocid = max(self.docpaths.keys()) except ValueError: self.maxdocid = 0 print len(self.docpaths), "Document ids" print len(self.path2docid), "Pathnames" print self.index.lexicon.length(), "Words"
def fixupPloneLexicon(context): """Updates the plone_lexicon pipeline with the new splitter and case normalizer. """ catalog = getToolByName(context, 'portal_catalog', None) if catalog is not None: if 'plone_lexicon' in catalog.objectIds(): lexicon = catalog.plone_lexicon pipeline = list(lexicon._pipeline) if len(pipeline) >= 2: if (not isinstance(pipeline[0], Splitter) or not isinstance(pipeline[1], CaseNormalizer)): pipeline[0] = Splitter() pipeline[1] = CaseNormalizer() lexicon._pipeline = tuple(pipeline) # Clear the lexicon from BTrees.OIBTree import OIBTree from BTrees.IOBTree import IOBTree from BTrees.Length import Length lexicon._wids = OIBTree() lexicon._words = IOBTree() lexicon.length = Length() logger.info('Updated plone_lexicon pipeline.')
def initialize_storage(self): ann = IAnnotations(self.context) if self.ANNOTATIONS_KEY not in ann: ann[self.ANNOTATIONS_KEY] = OOBTree() self._storage = ann[self.ANNOTATIONS_KEY] # Actual list of actions if self.STORAGE_ACTIONS_KEY not in self._storage: self._storage[self.STORAGE_ACTIONS_KEY] = IOBTree() self._actions = self._storage[self.STORAGE_ACTIONS_KEY] # Indexes needed for fast lookups if self.STORAGE_INDEXES_KEY not in self._storage: self._storage[self.STORAGE_INDEXES_KEY] = OOBTree() self._indexes = self._storage[self.STORAGE_INDEXES_KEY] # Index: unique_name -> action_id if self.IDX_UNIQUE_NAME not in self._indexes: self._indexes[self.IDX_UNIQUE_NAME] = OIBTree() # Counter for the next 'action_id' if self.STORAGE_NEXT_ID_KEY not in self._storage: self._storage[self.STORAGE_NEXT_ID_KEY] = 0
def create_token(self, userid, timeout=None, data=None): payload = {} payload["sub"] = userid if timeout is None: timeout = self.token_timeout if timeout: payload["exp"] = datetime.utcnow() + timedelta(seconds=timeout) if data is not None: payload.update(data) algorithm = "HS256" if self.haveRSAKeys(): algorithm = "RS256" token = jwt.encode(payload, self._signing_secret(), algorithm=algorithm) if not six.PY2: token = token.decode("utf-8") if self.store_tokens: if self._tokens is None: self._tokens = OOBTree() if userid not in self._tokens: self._tokens[userid] = OIBTree() self._tokens[userid][token] = int(time.time()) return token
def __init__(self): self.docid_to_address = IOBTree() self.address_to_docid = OIBTree() self.docid_to_metadata = IOBTree()
def __init__(self): """ Ballots attr is an OIBTree, since they can have any object as key. """ self.ballots = OIBTree()
def clear(self): """ Complete reset """ self._index = IOBTree() self._unindex = OIBTree() self._length = Length()