def _convertBTrees(self, threshold=200):
if type(self._lexicon) is type(''):
# Turn the name reference into a hard reference.
self._lexicon = self.getLexicon()
if type(self._index) is IOBTree: return
from BTrees.convert import convert
_index = self._index
self._index = IOBTree()
def convertScores(scores,
type=type,
TupleType=TupleType,
IIBTree=IIBTree):
if type(scores) is not TupleType and type(scores) is not IIBTree():
scores = IIBTree(scores)
return scores
convert(_index, self._index, threshold, convertScores)
_unindex = self._unindex
self._unindex = IOBTree()
convert(_unindex, self._unindex, threshold)
def __init__(self, lexicon):
self._lexicon = lexicon
# wid -> {docid -> weight}; t -> D -> w(D, t)
# Different indexers have different notions of term weight, but we
# expect each indexer to use ._wordinfo to map wids to its notion
# of a docid-to-weight map.
# There are two kinds of OOV words: wid 0 is explicitly OOV,
# and it's possible that the lexicon will return a non-zero wid
# for a word we don't currently know about. For example, if we
# unindex the last doc containing a particular word, that wid
# remains in the lexicon, but is no longer in our _wordinfo map;
# lexicons can also be shared across indices, and some other index
# may introduce a lexicon word we've never seen.
# A word is in-vocabulary for this index if and only if
# _wordinfo.has_key(wid). Note that wid 0 must not be a key.
self._wordinfo = IOBTree()
# docid -> weight
# Different indexers have different notions of doc weight, but we
# expect each indexer to use ._docweight to map docids to its
# notion of what a doc weight is.
self._docweight = IIBTree()
# docid -> WidCode'd list of wids
# Used for un-indexing, and for phrase search.
self._docwords = IOBTree()
# Use a BTree length for efficient length computation w/o conflicts
self.length = Length()
self.document_count = Length()
def addEdge(self, start, end, edgetype, **kwargs):
_id = self.edgeid()
if type(edgetype) != int:
edgetype = self.typeid(edgetype)
if type(start) == dict:
start = start['_id']
if type(end) == dict:
end = end['_id']
edge = [start, end, edgetype]
self.edges[_id] = edge
if kwargs:
self.edgedata[_id] = kwargs
le = self.lightEdge(_id, edge)
self.edge_catalog.index_doc(_id, le)
le = self.lightEdge(_id, edge)
# edgeid:nodeid
data = self.outgoing.setdefault(edgetype,
IOBTree()).setdefault(start, {})
data[_id] = end
self.outgoing[edgetype][start] = data
data = self.incoming.setdefault(edgetype,
IOBTree()).setdefault(end, {})
data[_id] = start
self.incoming[edgetype][end] = data
return le
def clear(self):
"""Reinitialize the text index."""
self._index = IOBTree()
self._unindex = IOBTree()
if self.getLexicon() and self.vocabulary_id == '__userdefined__':
self.getLexicon().clear()
self._lexicon = None
def __init__(self, id=None, **kwargs):
super(TracListing, self).__init__(id, **kwargs)
# manage an index of parent-to-child, int parent ticket id key
# to PersistentList of int ticket id value:
self._children = IOBTree()
# indexes for score and reward-ratio values:
self._scores = IIBTree() # int (ticket#) -> int (sum/score)
self._reward = IOBTree() # int (ticket#) -> float (ratio)
def clear(self):
"""Start over fresh."""
self._always = IITreeSet()
self._since_only = IOBTree()
self._until_only = IOBTree()
self._since = IOBTree()
self._until = IOBTree()
self._unindex = IOBTree() # 'datum' will be a tuple of date ints
self._length = Length()
def initializeRoomBookingDB(location, force=False):
"""
Modifies Indico main database.
- location list
Creates necessary branches in room booking database.
- rooms branch
- bookings branch
- several indexes
"""
indicoRoot = MaKaC.common.DBMgr.getInstance().getDBConnection().root()
root = DALManagerCERN().getRoot()
# 1. Location -----------------------------------------------------------
initialLocation = Location(location, FactoryCERN)
if force or not indicoRoot.has_key('RoomBookingLocationList'):
indicoRoot['RoomBookingLocationList'] = [initialLocation]
if force or not indicoRoot.has_key('DefaultRoomBookingLocation'):
indicoRoot['DefaultRoomBookingLocation'] = location
# 2. Rooms & Bookings ---------------------------------------------------
# Create rooms branch
if force or not root.has_key('Rooms'):
root['Rooms'] = IOBTree()
# Create reservations branch
if force or not root.has_key('Reservations'):
root['Reservations'] = IOBTree()
# Create blocking branch
if force or not root.has_key('RoomBlocking'):
root['RoomBlocking'] = OOBTree()
root['RoomBlocking']['Blockings'] = IOBTree()
root['RoomBlocking']['Indexes'] = OOBTree()
root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree()
root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex()
root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree()
# Create indexes
if force or not root.has_key('RoomReservationsIndex'):
root['RoomReservationsIndex'] = OOBTree()
if force or not root.has_key('UserReservationsIndex'):
root['UserReservationsIndex'] = OOBTree()
if force or not root.has_key('DayReservationsIndex'):
root['DayReservationsIndex'] = OOBTree()
if force or not root.has_key('RoomDayReservationsIndex'):
root['RoomDayReservationsIndex'] = OOBTree()
# Create possible equipment branch
if force or not root.has_key('EquipmentList'):
root['EquipmentList'] = {}
# update Catalog with new rb indexes
Catalog.updateDB()
DALManagerCERN().commit()
def clear(self):
""" clear catalog """
self.data = IOBTree() # mapping of rid to meta_data
self.uids = OIBTree() # mapping of uid to rid
self.paths = IOBTree() # mapping of rid to uid
self._length = BTrees.Length.Length()
for index in self.indexes.keys():
self.getIndex(index).clear()
def initializeRoomBookingDB(location, force=False):
"""
Modifies Indico main database.
- location list
Creates necessary branches in room booking database.
- rooms branch
- bookings branch
- several indexes
"""
indicoRoot = MaKaC.common.DBMgr.getInstance().getDBConnection().root()
root = DALManagerCERN().root
# 1. Location -----------------------------------------------------------
initialLocation = Location(location, FactoryCERN)
if force or not indicoRoot.has_key('RoomBookingLocationList'):
indicoRoot['RoomBookingLocationList'] = [initialLocation]
print "Locations branch created (Indico db)"
if force or not indicoRoot.has_key('DefaultRoomBookingLocation'):
indicoRoot['DefaultRoomBookingLocation'] = location
print "Default location set to " + location
# 2. Rooms & Bookings ---------------------------------------------------
# Create rooms branch
if force or not root.has_key('Rooms'):
root['Rooms'] = IOBTree()
print "Rooms branch created"
# Create reservations branch
if force or not root.has_key('Reservations'):
root['Reservations'] = IOBTree()
print "Reservations branch created"
# Create indexes
if force or not root.has_key('RoomReservationsIndex'):
root['RoomReservationsIndex'] = OOBTree()
print "Room => Reservations Index branch created"
if force or not root.has_key('UserReservationsIndex'):
root['UserReservationsIndex'] = OOBTree()
print "User => Reservations Index branch created"
if force or not root.has_key('DayReservationsIndex'):
root['DayReservationsIndex'] = OOBTree()
print "Day => Reservations Index branch created"
# Create possible equipment branch
if force or not root.has_key('EquipmentList'):
root['EquipmentList'] = {}
print "Equipment branch created"
AvatarHolder().invalidateRoomManagerIdList()
print "Cached list of room managers invalidated"
DALManagerCERN().commit()
def _init(self):
self.nodes = IOBTree()
self.edges = IOBTree()
self.edgedata = IOBTree()
self.outgoing = IOBTree()
self.incoming = IOBTree()
self.typeids = PObject()
self._nodeid = Length(0)
self._edgeid = Length(0)
self._typeid = Length(0)
def clear(self):
"""Start over fresh."""
self._always = IITreeSet()
self._since_only = IOBTree()
self._until_only = IOBTree()
self._since = IOBTree()
self._until = IOBTree()
self._unindex = IOBTree() # 'datum' will be a tuple of date ints
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def __init__(self):
curdir = os.path.abspath(os.path.dirname(__file__))
db_file = os.path.join(curdir, 'OData.db')
storage = FileStorage.FileStorage(db_file)
db = DB(storage)
conn = db.open()
self.dbroot = conn.root()
if 'zcalibdb' not in self.dbroot:
self.dbroot['zcalibdb'] = OOBTree()
zcalibdb = self.dbroot['zcalibdb']
zcalibdb['members'] = IOBTree()
zcalibdb['books'] = IOBTree()
zcalibdb['circulations'] = IOBTree()
self.commit()
def createEvent(self,
obj,
date,
eventType=None,
eventSpan=None,
handleEventSpan=True):
'''Create a new event in the calendar, at some p_date (day).
If p_eventType is given, it is used; else, rq['eventType'] is used.
If p_handleEventSpan is True, we will use p_eventSpan (or
rq["eventSpan"] if p_eventSpan is not given) and also
create the same event for successive days.'''
obj = obj.o # Ensure p_obj is not a wrapper.
rq = obj.REQUEST
# Get values from parameters
if not eventType: eventType = rq['eventType']
if handleEventSpan and not eventSpan:
eventSpan = rq.get('eventSpan', None)
# Split the p_date into separate parts
year, month, day = date.year(), date.month(), date.day()
# Check that the "preferences" dict exists or not.
if not hasattr(obj.aq_base, self.name):
# 1st level: create a IOBTree whose keys are years.
setattr(obj, self.name, IOBTree())
yearsDict = getattr(obj, self.name)
# Get the sub-dict storing months for a given year
if year in yearsDict:
monthsDict = yearsDict[year]
else:
yearsDict[year] = monthsDict = IOBTree()
# Get the sub-dict storing days of a given month
if month in monthsDict:
daysDict = monthsDict[month]
else:
monthsDict[month] = daysDict = IOBTree()
# Get the list of events for a given day
if day in daysDict:
events = daysDict[day]
else:
daysDict[day] = events = PersistentList()
# Create and store the event, excepted if an event already exists.
if not events:
event = Object(eventType=eventType)
events.append(event)
# Span the event on the successive days if required
if handleEventSpan and eventSpan:
nbOfDays = min(int(eventSpan), self.maxEventLength)
for i in range(nbOfDays):
date = date + 1
self.createEvent(obj, date, handleEventSpan=False)
def _mapFromParams(self, params):
self._normaliseAuthorParams(params)
self._primaryAuthors = IOBTree()
self._secondaryAuthors = IOBTree()
maxId = -1
for idx in range(len(params["auth_id"])):
id = int(params["auth_id"][idx])
if id > maxId:
maxId = id
author = self._getAuthorFromParams(idx, params)
if author["auth_primary"]:
self._primaryAuthors[id] = author
else:
self._secondaryAuthors[id] = author
self._nextId = maxId + 1
def add(self, language, value, key):
self._fixup()
tree = self.data.get(language)
if tree is None:
tree = self.data[language] = OOBTree()
else:
# Make sure we update the modification time.
self.data[language] = tree
update = key in tree
tree[key] = value
order = self.order.get(language)
if order is None:
order = self.order[language] = IOBTree()
count = self.count[language] = 0
else:
if update:
pop_value(tree, key)
count = self.count[language] + 1
self.count[language] = count
order[count] = key
def __init__(self):
"""Setup our data structures"""
self._anon_ratings = IOBTree()
self._ratings = OOBTree()
self._sessions = OOBTree()
self._length = Length()
self._anon_length = Length()
def clear(self):
self.data = IOBTree() # {rid -> record as tuple}
self.indexes = {} # {index_name -> OOBTree({value -> IITreeSet})}
self.primary_index = OIBTree() # {primary key -> rid}
for position, column in self.col_info:
if column.indexed:
self.indexes[column.name] = OOBTree()
def clear(self):
"""Empty the lexicon.
"""
self.length = Length()
self._wid_length_based = False
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
def clear(self):
""" See IPluggableIndex.
"""
self._depth = 0
self._index = OOBTree()
self._unindex = IOBTree()
self._length = Length(0)
def refresh(self):
"""Refresh the data stored.
"""
if self._data is None:
self._data = IOBTree()
counts = {}
catalog = getUtility(ICatalogService)
for brain in catalog(meta_type=['Silva Software Activity'],
path='/'.join(
self.get_container().getPhysicalPath())):
activity = brain.getObject()
counts[brain.content_intid] = 0
changes = activity.get_changes_since(empty=False)
print activity.get_title(), len(changes)
for change in changes:
for commit in change:
key = commit.date.toordinal()
if key not in self._data:
self._data[key] = Changes()
self._data[key].add(commit)
counts[brain.content_intid] += len(change)
self._most_actives = map(
operator.itemgetter(0),
sorted(counts.items(), key=operator.itemgetter(1), reverse=True))
def __init__(self):
ouiPath = os.getenv("ICT_OUI_PATH")
if ouiPath is not None:
f = open(ouiPath, "r") #oui: http://standards.ieee.org/regauth/oui/oui.txt
else:
f = open("src/org/ict_ok/libs/oui.txt", "r") #oui: http://standards.ieee.org/regauth/oui/oui.txt
self._data = IOBTree()
last_mac = None
EOF = False
re_mac = re.compile(r"[A-F0-9]*\-[A-F0-9]*\-[A-F0-9]*")
while not EOF:
line = f.readline()
if re.match(re_mac, line):
last_mac = int(line[:8].replace("-", ""), 16)
index = line.rfind("\t")+len("\t")
org = line[index:].strip()
self._data[last_mac] = {"short": org, "other":[]}
elif last_mac != None:
index = line.rfind("\t")
if index != -1:
index += len("\t")
other = line[index:].strip()
self._data[last_mac]["other"].append(other)
if (line == "\n"):
country_index = len(self._data[last_mac]["other"])-1 #last element
self._data[last_mac]["country"] = self._data[last_mac]["other"][country_index]
del self._data[last_mac]["other"][country_index]
if line == '':
EOF=True
f.close()
def assignWordId(self, word):
"""Assigns a new word id to the provided word and returns it."""
# First make sure it's not already in there
if self._lexicon.has_key(word):
return self._lexicon[word]
try: inverse=self._inverseLex
except AttributeError:
# woops, old lexicom wo wids
inverse=self._inverseLex=IOBTree()
for word, wid in self._lexicon.items():
inverse[wid]=word
wid=randid()
while not inverse.insert(wid, word):
wid=randid()
if isinstance(word,StringType):
self._lexicon[intern(word)] = wid
else:
self._lexicon[word] = wid
return wid
def _migrate_event_reviewing(self, conf):
conference_settings = getattr(conf, '_confPaperReview', None)
if not conference_settings:
return
event = conf.as_event
contrib_index = conference_settings._contribution_index = IOBTree()
contrib_reviewers = _invert_mapping(conference_settings._reviewerContribution)
contrib_referees = _invert_mapping(conference_settings._refereeContribution)
contrib_editors = _invert_mapping(conference_settings._editorContribution)
for old_contribution in conf.contributions.itervalues():
review_manager = getattr(old_contribution, '_reviewManager', None)
new_contribution = Contribution.find_one(event_id=event.id, friendly_id=int(old_contribution.id))
cid = int(new_contribution.id)
if review_manager:
review_manager._contrib_id = cid
contrib_index[cid] = review_manager
self._migrate_contribution_roles(old_contribution, new_contribution, contrib_reviewers,
PaperReviewingRoleType.reviewer, event.id)
self._migrate_contribution_roles(old_contribution, new_contribution, contrib_referees,
PaperReviewingRoleType.referee, event.id)
self._migrate_contribution_roles(old_contribution, new_contribution, contrib_editors,
PaperReviewingRoleType.editor, event.id)
self._migrate_reviewing_materials(old_contribution, new_contribution, review_manager, event.id)
def insert(self, idx, results, relnames=None, treePrefix=None):
for brain in results:
rid = brain.getRID()
path = brain.getPath()
if treePrefix and not path.startswith(treePrefix):
paths = brain.global_catalog._catalog.indexes['path']._unindex[
rid]
for p in paths:
if p.startswith(treePrefix):
path = p
break
else:
paths = [path]
for path in paths:
path = path.split('/', 3)[-1]
if relnames:
if isinstance(relnames, basestring):
relnames = (relnames, )
for relname in relnames:
path = path.replace('/' + relname, '')
if rid: # TODO review this I just did it to avoid exception
self._brains[rid] = brain
for depth in xrange(path.count('/') + 1):
comp = idx.setdefault(path, IOBTree())
comp.setdefault(depth, []).append(rid)
path = path.rsplit('/', 1)[0]
def clear(self):
self._length = Length()
self._index = OIBTree()
self._unindex = IOBTree()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def __init__(self):
self.timeouts = IOBTree()
self.locks = OOBTree()
if self.__class__ is LockStorage:
raise TypeError(
"%s.LockStorage is insane; use a persistent subclass instead"
" (%s.PersistentLockStorage should do nicely)"
% (__name__, __name__))
def clear(self):
""" See IPluggableIndex.
"""
self.numObjects = Length()
# Mapping field value => top items
self._value2items = OOBTree()
# Mapping indexed rid => field value for unindexing
self._rid2value = IOBTree()
def clear(self):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def __init__(self, storage, db=None, scan_interval=10):
self.storage = storage
self.db = db
self.next_conn_id = 1
self.conn_oids = IOBTree() # IOBTree({ conn_id -> OOSet([oid]) } })
self.oids = OOSet() # OOSet([oid])
self.lock = allocate_lock()
self.scan_interval = scan_interval
self.next_scan = time() + scan_interval
def index(rt, mboxfile, db, profiler):
global NUM
idx_time = 0
pack_time = 0
start_time = time.time()
lexicon = Lexicon(Splitter(), CaseNormalizer(), StopWordRemover())
extra = Extra()
extra.lexicon_id = 'lexicon'
extra.doc_attr = 'text'
extra.index_type = 'Okapi BM25 Rank'
caller = Extra()
caller.lexicon = lexicon
rt["index"] = idx = ZCTextIndex("index", extra, caller)
if not EXCLUDE_TEXT:
rt["documents"] = docs = IOBTree()
else:
docs = None
transaction.commit()
mbox = mailbox.UnixMailbox(open(mboxfile, 'rb'))
if VERBOSE:
print "opened", mboxfile
if not NUM:
NUM = sys.maxint
if profiler:
itime, ptime, i = profiler.runcall(indexmbox, mbox, idx, docs, db)
else:
itime, ptime, i = indexmbox(mbox, idx, docs, db)
idx_time += itime
pack_time += ptime
transaction.commit()
if PACK_INTERVAL and i % PACK_INTERVAL != 0:
if VERBOSE >= 2:
print "packing one last time..."
p0 = time.clock()
db.pack(time.time())
p1 = time.clock()
if VERBOSE:
print "pack took %s sec" % (p1 - p0)
pack_time += p1 - p0
if VERBOSE:
finish_time = time.time()
print
print "Index time", round(idx_time / 60, 3), "minutes"
print "Pack time", round(pack_time / 60, 3), "minutes"
print "Index bytes", Message.total_bytes
rate = (Message.total_bytes / idx_time) / 1024
print "Index rate %.2f KB/sec" % rate
print "Indexing began", time.ctime(start_time)
print "Indexing ended", time.ctime(finish_time)
print "Wall clock minutes", round((finish_time - start_time) / 60, 3)
