def _convertBTrees(self, threshold=200): if type(self._lexicon) is type(''): # Turn the name reference into a hard reference. self._lexicon = self.getLexicon() if type(self._index) is IOBTree: return from BTrees.convert import convert _index = self._index self._index = IOBTree() def convertScores(scores, type=type, TupleType=TupleType, IIBTree=IIBTree): if type(scores) is not TupleType and type(scores) is not IIBTree(): scores = IIBTree(scores) return scores convert(_index, self._index, threshold, convertScores) _unindex = self._unindex self._unindex = IOBTree() convert(_unindex, self._unindex, threshold)
def __init__(self, lexicon): self._lexicon = lexicon # wid -> {docid -> weight}; t -> D -> w(D, t) # Different indexers have different notions of term weight, but we # expect each indexer to use ._wordinfo to map wids to its notion # of a docid-to-weight map. # There are two kinds of OOV words: wid 0 is explicitly OOV, # and it's possible that the lexicon will return a non-zero wid # for a word we don't currently know about. For example, if we # unindex the last doc containing a particular word, that wid # remains in the lexicon, but is no longer in our _wordinfo map; # lexicons can also be shared across indices, and some other index # may introduce a lexicon word we've never seen. # A word is in-vocabulary for this index if and only if # _wordinfo.has_key(wid). Note that wid 0 must not be a key. self._wordinfo = IOBTree() # docid -> weight # Different indexers have different notions of doc weight, but we # expect each indexer to use ._docweight to map docids to its # notion of what a doc weight is. self._docweight = IIBTree() # docid -> WidCode'd list of wids # Used for un-indexing, and for phrase search. self._docwords = IOBTree() # Use a BTree length for efficient length computation w/o conflicts self.length = Length() self.document_count = Length()
def addEdge(self, start, end, edgetype, **kwargs): _id = self.edgeid() if type(edgetype) != int: edgetype = self.typeid(edgetype) if type(start) == dict: start = start['_id'] if type(end) == dict: end = end['_id'] edge = [start, end, edgetype] self.edges[_id] = edge if kwargs: self.edgedata[_id] = kwargs le = self.lightEdge(_id, edge) self.edge_catalog.index_doc(_id, le) le = self.lightEdge(_id, edge) # edgeid:nodeid data = self.outgoing.setdefault(edgetype, IOBTree()).setdefault(start, {}) data[_id] = end self.outgoing[edgetype][start] = data data = self.incoming.setdefault(edgetype, IOBTree()).setdefault(end, {}) data[_id] = start self.incoming[edgetype][end] = data return le
def clear(self): """Reinitialize the text index.""" self._index = IOBTree() self._unindex = IOBTree() if self.getLexicon() and self.vocabulary_id == '__userdefined__': self.getLexicon().clear() self._lexicon = None
def __init__(self, id=None, **kwargs): super(TracListing, self).__init__(id, **kwargs) # manage an index of parent-to-child, int parent ticket id key # to PersistentList of int ticket id value: self._children = IOBTree() # indexes for score and reward-ratio values: self._scores = IIBTree() # int (ticket#) -> int (sum/score) self._reward = IOBTree() # int (ticket#) -> float (ratio)
def clear(self): """Start over fresh.""" self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length()
def initializeRoomBookingDB(location, force=False): """ Modifies Indico main database. - location list Creates necessary branches in room booking database. - rooms branch - bookings branch - several indexes """ indicoRoot = MaKaC.common.DBMgr.getInstance().getDBConnection().root() root = DALManagerCERN().getRoot() # 1. Location ----------------------------------------------------------- initialLocation = Location(location, FactoryCERN) if force or not indicoRoot.has_key('RoomBookingLocationList'): indicoRoot['RoomBookingLocationList'] = [initialLocation] if force or not indicoRoot.has_key('DefaultRoomBookingLocation'): indicoRoot['DefaultRoomBookingLocation'] = location # 2. Rooms & Bookings --------------------------------------------------- # Create rooms branch if force or not root.has_key('Rooms'): root['Rooms'] = IOBTree() # Create reservations branch if force or not root.has_key('Reservations'): root['Reservations'] = IOBTree() # Create blocking branch if force or not root.has_key('RoomBlocking'): root['RoomBlocking'] = OOBTree() root['RoomBlocking']['Blockings'] = IOBTree() root['RoomBlocking']['Indexes'] = OOBTree() root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree() root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex() root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree() # Create indexes if force or not root.has_key('RoomReservationsIndex'): root['RoomReservationsIndex'] = OOBTree() if force or not root.has_key('UserReservationsIndex'): root['UserReservationsIndex'] = OOBTree() if force or not root.has_key('DayReservationsIndex'): root['DayReservationsIndex'] = OOBTree() if force or not root.has_key('RoomDayReservationsIndex'): root['RoomDayReservationsIndex'] = OOBTree() # Create possible equipment branch if force or not root.has_key('EquipmentList'): root['EquipmentList'] = {} # update Catalog with new rb indexes Catalog.updateDB() DALManagerCERN().commit()
def clear(self): """ clear catalog """ self.data = IOBTree() # mapping of rid to meta_data self.uids = OIBTree() # mapping of uid to rid self.paths = IOBTree() # mapping of rid to uid self._length = BTrees.Length.Length() for index in self.indexes.keys(): self.getIndex(index).clear()
def initializeRoomBookingDB(location, force=False): """ Modifies Indico main database. - location list Creates necessary branches in room booking database. - rooms branch - bookings branch - several indexes """ indicoRoot = MaKaC.common.DBMgr.getInstance().getDBConnection().root() root = DALManagerCERN().root # 1. Location ----------------------------------------------------------- initialLocation = Location(location, FactoryCERN) if force or not indicoRoot.has_key('RoomBookingLocationList'): indicoRoot['RoomBookingLocationList'] = [initialLocation] print "Locations branch created (Indico db)" if force or not indicoRoot.has_key('DefaultRoomBookingLocation'): indicoRoot['DefaultRoomBookingLocation'] = location print "Default location set to " + location # 2. Rooms & Bookings --------------------------------------------------- # Create rooms branch if force or not root.has_key('Rooms'): root['Rooms'] = IOBTree() print "Rooms branch created" # Create reservations branch if force or not root.has_key('Reservations'): root['Reservations'] = IOBTree() print "Reservations branch created" # Create indexes if force or not root.has_key('RoomReservationsIndex'): root['RoomReservationsIndex'] = OOBTree() print "Room => Reservations Index branch created" if force or not root.has_key('UserReservationsIndex'): root['UserReservationsIndex'] = OOBTree() print "User => Reservations Index branch created" if force or not root.has_key('DayReservationsIndex'): root['DayReservationsIndex'] = OOBTree() print "Day => Reservations Index branch created" # Create possible equipment branch if force or not root.has_key('EquipmentList'): root['EquipmentList'] = {} print "Equipment branch created" AvatarHolder().invalidateRoomManagerIdList() print "Cached list of room managers invalidated" DALManagerCERN().commit()
def _init(self): self.nodes = IOBTree() self.edges = IOBTree() self.edgedata = IOBTree() self.outgoing = IOBTree() self.incoming = IOBTree() self.typeids = PObject() self._nodeid = Length(0) self._edgeid = Length(0) self._typeid = Length(0)
def clear(self): """Start over fresh.""" self._always = IITreeSet() self._since_only = IOBTree() self._until_only = IOBTree() self._since = IOBTree() self._until = IOBTree() self._unindex = IOBTree() # 'datum' will be a tuple of date ints self._length = Length() if self._counter is None: self._counter = Length() else: self._increment_counter()
def __init__(self): curdir = os.path.abspath(os.path.dirname(__file__)) db_file = os.path.join(curdir, 'OData.db') storage = FileStorage.FileStorage(db_file) db = DB(storage) conn = db.open() self.dbroot = conn.root() if 'zcalibdb' not in self.dbroot: self.dbroot['zcalibdb'] = OOBTree() zcalibdb = self.dbroot['zcalibdb'] zcalibdb['members'] = IOBTree() zcalibdb['books'] = IOBTree() zcalibdb['circulations'] = IOBTree() self.commit()
def createEvent(self, obj, date, eventType=None, eventSpan=None, handleEventSpan=True): '''Create a new event in the calendar, at some p_date (day). If p_eventType is given, it is used; else, rq['eventType'] is used. If p_handleEventSpan is True, we will use p_eventSpan (or rq["eventSpan"] if p_eventSpan is not given) and also create the same event for successive days.''' obj = obj.o # Ensure p_obj is not a wrapper. rq = obj.REQUEST # Get values from parameters if not eventType: eventType = rq['eventType'] if handleEventSpan and not eventSpan: eventSpan = rq.get('eventSpan', None) # Split the p_date into separate parts year, month, day = date.year(), date.month(), date.day() # Check that the "preferences" dict exists or not. if not hasattr(obj.aq_base, self.name): # 1st level: create a IOBTree whose keys are years. setattr(obj, self.name, IOBTree()) yearsDict = getattr(obj, self.name) # Get the sub-dict storing months for a given year if year in yearsDict: monthsDict = yearsDict[year] else: yearsDict[year] = monthsDict = IOBTree() # Get the sub-dict storing days of a given month if month in monthsDict: daysDict = monthsDict[month] else: monthsDict[month] = daysDict = IOBTree() # Get the list of events for a given day if day in daysDict: events = daysDict[day] else: daysDict[day] = events = PersistentList() # Create and store the event, excepted if an event already exists. if not events: event = Object(eventType=eventType) events.append(event) # Span the event on the successive days if required if handleEventSpan and eventSpan: nbOfDays = min(int(eventSpan), self.maxEventLength) for i in range(nbOfDays): date = date + 1 self.createEvent(obj, date, handleEventSpan=False)
def _mapFromParams(self, params): self._normaliseAuthorParams(params) self._primaryAuthors = IOBTree() self._secondaryAuthors = IOBTree() maxId = -1 for idx in range(len(params["auth_id"])): id = int(params["auth_id"][idx]) if id > maxId: maxId = id author = self._getAuthorFromParams(idx, params) if author["auth_primary"]: self._primaryAuthors[id] = author else: self._secondaryAuthors[id] = author self._nextId = maxId + 1
def add(self, language, value, key): self._fixup() tree = self.data.get(language) if tree is None: tree = self.data[language] = OOBTree() else: # Make sure we update the modification time. self.data[language] = tree update = key in tree tree[key] = value order = self.order.get(language) if order is None: order = self.order[language] = IOBTree() count = self.count[language] = 0 else: if update: pop_value(tree, key) count = self.count[language] + 1 self.count[language] = count order[count] = key
def __init__(self): """Setup our data structures""" self._anon_ratings = IOBTree() self._ratings = OOBTree() self._sessions = OOBTree() self._length = Length() self._anon_length = Length()
def clear(self): self.data = IOBTree() # {rid -> record as tuple} self.indexes = {} # {index_name -> OOBTree({value -> IITreeSet})} self.primary_index = OIBTree() # {primary key -> rid} for position, column in self.col_info: if column.indexed: self.indexes[column.name] = OOBTree()
def clear(self): """Empty the lexicon. """ self.length = Length() self._wid_length_based = False self._wids = OIBTree() # word -> wid self._words = IOBTree() # wid -> word
def clear(self): """ See IPluggableIndex. """ self._depth = 0 self._index = OOBTree() self._unindex = IOBTree() self._length = Length(0)
def refresh(self): """Refresh the data stored. """ if self._data is None: self._data = IOBTree() counts = {} catalog = getUtility(ICatalogService) for brain in catalog(meta_type=['Silva Software Activity'], path='/'.join( self.get_container().getPhysicalPath())): activity = brain.getObject() counts[brain.content_intid] = 0 changes = activity.get_changes_since(empty=False) print activity.get_title(), len(changes) for change in changes: for commit in change: key = commit.date.toordinal() if key not in self._data: self._data[key] = Changes() self._data[key].add(commit) counts[brain.content_intid] += len(change) self._most_actives = map( operator.itemgetter(0), sorted(counts.items(), key=operator.itemgetter(1), reverse=True))
def __init__(self): ouiPath = os.getenv("ICT_OUI_PATH") if ouiPath is not None: f = open(ouiPath, "r") #oui: http://standards.ieee.org/regauth/oui/oui.txt else: f = open("src/org/ict_ok/libs/oui.txt", "r") #oui: http://standards.ieee.org/regauth/oui/oui.txt self._data = IOBTree() last_mac = None EOF = False re_mac = re.compile(r"[A-F0-9]*\-[A-F0-9]*\-[A-F0-9]*") while not EOF: line = f.readline() if re.match(re_mac, line): last_mac = int(line[:8].replace("-", ""), 16) index = line.rfind("\t")+len("\t") org = line[index:].strip() self._data[last_mac] = {"short": org, "other":[]} elif last_mac != None: index = line.rfind("\t") if index != -1: index += len("\t") other = line[index:].strip() self._data[last_mac]["other"].append(other) if (line == "\n"): country_index = len(self._data[last_mac]["other"])-1 #last element self._data[last_mac]["country"] = self._data[last_mac]["other"][country_index] del self._data[last_mac]["other"][country_index] if line == '': EOF=True f.close()
def assignWordId(self, word): """Assigns a new word id to the provided word and returns it.""" # First make sure it's not already in there if self._lexicon.has_key(word): return self._lexicon[word] try: inverse=self._inverseLex except AttributeError: # woops, old lexicom wo wids inverse=self._inverseLex=IOBTree() for word, wid in self._lexicon.items(): inverse[wid]=word wid=randid() while not inverse.insert(wid, word): wid=randid() if isinstance(word,StringType): self._lexicon[intern(word)] = wid else: self._lexicon[word] = wid return wid
def _migrate_event_reviewing(self, conf): conference_settings = getattr(conf, '_confPaperReview', None) if not conference_settings: return event = conf.as_event contrib_index = conference_settings._contribution_index = IOBTree() contrib_reviewers = _invert_mapping(conference_settings._reviewerContribution) contrib_referees = _invert_mapping(conference_settings._refereeContribution) contrib_editors = _invert_mapping(conference_settings._editorContribution) for old_contribution in conf.contributions.itervalues(): review_manager = getattr(old_contribution, '_reviewManager', None) new_contribution = Contribution.find_one(event_id=event.id, friendly_id=int(old_contribution.id)) cid = int(new_contribution.id) if review_manager: review_manager._contrib_id = cid contrib_index[cid] = review_manager self._migrate_contribution_roles(old_contribution, new_contribution, contrib_reviewers, PaperReviewingRoleType.reviewer, event.id) self._migrate_contribution_roles(old_contribution, new_contribution, contrib_referees, PaperReviewingRoleType.referee, event.id) self._migrate_contribution_roles(old_contribution, new_contribution, contrib_editors, PaperReviewingRoleType.editor, event.id) self._migrate_reviewing_materials(old_contribution, new_contribution, review_manager, event.id)
def insert(self, idx, results, relnames=None, treePrefix=None): for brain in results: rid = brain.getRID() path = brain.getPath() if treePrefix and not path.startswith(treePrefix): paths = brain.global_catalog._catalog.indexes['path']._unindex[ rid] for p in paths: if p.startswith(treePrefix): path = p break else: paths = [path] for path in paths: path = path.split('/', 3)[-1] if relnames: if isinstance(relnames, basestring): relnames = (relnames, ) for relname in relnames: path = path.replace('/' + relname, '') if rid: # TODO review this I just did it to avoid exception self._brains[rid] = brain for depth in xrange(path.count('/') + 1): comp = idx.setdefault(path, IOBTree()) comp.setdefault(depth, []).append(rid) path = path.rsplit('/', 1)[0]
def clear(self): self._length = Length() self._index = OIBTree() self._unindex = IOBTree() if self._counter is None: self._counter = Length() else: self._increment_counter()
def __init__(self): self.timeouts = IOBTree() self.locks = OOBTree() if self.__class__ is LockStorage: raise TypeError( "%s.LockStorage is insane; use a persistent subclass instead" " (%s.PersistentLockStorage should do nicely)" % (__name__, __name__))
def clear(self): """ See IPluggableIndex. """ self.numObjects = Length() # Mapping field value => top items self._value2items = OOBTree() # Mapping indexed rid => field value for unindexing self._rid2value = IOBTree()
def clear(self): """ Complete reset """ self._index = IOBTree() self._unindex = IIBTree() self._length = Length() if self._counter is None: self._counter = Length() else: self._increment_counter()
def __init__(self, storage, db=None, scan_interval=10): self.storage = storage self.db = db self.next_conn_id = 1 self.conn_oids = IOBTree() # IOBTree({ conn_id -> OOSet([oid]) } }) self.oids = OOSet() # OOSet([oid]) self.lock = allocate_lock() self.scan_interval = scan_interval self.next_scan = time() + scan_interval
def index(rt, mboxfile, db, profiler): global NUM idx_time = 0 pack_time = 0 start_time = time.time() lexicon = Lexicon(Splitter(), CaseNormalizer(), StopWordRemover()) extra = Extra() extra.lexicon_id = 'lexicon' extra.doc_attr = 'text' extra.index_type = 'Okapi BM25 Rank' caller = Extra() caller.lexicon = lexicon rt["index"] = idx = ZCTextIndex("index", extra, caller) if not EXCLUDE_TEXT: rt["documents"] = docs = IOBTree() else: docs = None transaction.commit() mbox = mailbox.UnixMailbox(open(mboxfile, 'rb')) if VERBOSE: print "opened", mboxfile if not NUM: NUM = sys.maxint if profiler: itime, ptime, i = profiler.runcall(indexmbox, mbox, idx, docs, db) else: itime, ptime, i = indexmbox(mbox, idx, docs, db) idx_time += itime pack_time += ptime transaction.commit() if PACK_INTERVAL and i % PACK_INTERVAL != 0: if VERBOSE >= 2: print "packing one last time..." p0 = time.clock() db.pack(time.time()) p1 = time.clock() if VERBOSE: print "pack took %s sec" % (p1 - p0) pack_time += p1 - p0 if VERBOSE: finish_time = time.time() print print "Index time", round(idx_time / 60, 3), "minutes" print "Pack time", round(pack_time / 60, 3), "minutes" print "Index bytes", Message.total_bytes rate = (Message.total_bytes / idx_time) / 1024 print "Index rate %.2f KB/sec" % rate print "Indexing began", time.ctime(start_time) print "Indexing ended", time.ctime(finish_time) print "Wall clock minutes", round((finish_time - start_time) / 60, 3)