def migrateCategoryDateIndex():
IndexesHolder()._getIdx()["backupCategoryDate"] = IndexesHolder().getIndex(
"categoryDate")
newIdx = CategoryDayIndex()
newIdx.buildIndex()
IndexesHolder()._getIdx()["categoryDate"] = newIdx
print "Migration was successful"
def categoryDateIndexWithoutVisibility(dbi, withRBDB, prevVersion):
"""
Create category date index without visibility.
"""
IndexesHolder()._getIdx()['categoryDate']._useVisibility = True
if 'categoryDateAll' not in IndexesHolder()._getIdx():
newIdx = CategoryDayIndex(visibility=False)
IndexesHolder()._getIdx()['categoryDateAll'] = newIdx
newIdx.buildIndex(dbi)
def reindex(fix=False, fromDate=None):
""" Recreate deleted obj indices, from the DOH """
dbi = DBMgr.getInstance()
dbi.startRequest()
pubConfIdx = IndexesHolder().getIndex(
'OAIDeletedConferenceModificationDate')
prvConfIdx = IndexesHolder().getIndex(
'OAIDeletedPrivateConferenceModificationDate')
pubContIdx = IndexesHolder().getIndex(
'OAIDeletedContributionModificationDate')
prvContIdx = IndexesHolder().getIndex(
'OAIDeletedPrivateContributionModificationDate')
doh = DeletedObjectHolder()
pubConfIdx.initIndex()
pubContIdx.initIndex()
prvConfIdx.initIndex()
prvContIdx.initIndex()
fromDateParsed_tz = datetime.datetime(
*time.strptime(fromDate, '%Y-%m-%d')[:6],
**{'tzinfo': timezone('UTC')})
fromDateParsed_naive = datetime.datetime(
*time.strptime(fromDate, '%Y-%m-%d')[:6])
for obj in doh.getList():
if fromDate:
if obj.getOAIModificationDate().tzinfo:
fromDateParsed = fromDateParsed_tz
else:
fromDateParsed = fromDateParsed_naive
if obj.getOAIModificationDate() < fromDateParsed:
continue
if not hasattr(obj, 'protected'):
print "NO DATA FOR %s (%s)" % (obj.getId(),
obj.getOAIModificationDate())
continue
print "indexing %s (%s)" % (prettyPrint(obj),
obj.getOAIModificationDate())
if obj._objClass == Conference:
if obj.protected:
prvConfIdx.indexConference(obj)
else:
pubConfIdx.indexConference(obj)
elif obj._objClass == Contribution or obj._objClass == AcceptedContribution or obj._objClass == SubContribution:
if obj.protected:
prvContIdx.indexContribution(obj)
else:
pubContIdx.indexContribution(obj)
dbi.commit()
dbi.endRequest()
def setUp(self):
DBMgr.getInstance().startRequest()
self.oldIndex = IndexesHolder()._getIdx()["categoryDateLtd"]
self.newIndex = IndexesHolder()._getIdx()["categoryDate"]
self.startDate = datetime(2010,
5,
13,
10,
0,
0,
tzinfo=timezone('UTC'))
self.endDate = datetime(2010, 5, 14, 14, 0, 0, tzinfo=timezone('UTC'))
self.ch = ConferenceHolder()
self.categId = '0'
def reindexCategoryNameAndConferenceTitle(dbi, withRBDB, prevVersion):
"""
Indexing Conference Title with new WhooshTextIndex
"""
IndexesHolder().removeById('conferenceTitle')
IndexesHolder().removeById('categoryName')
confTitleIdx = IndexesHolder().getIndex('conferenceTitle')
categNameIdx = IndexesHolder().getIndex('categoryName')
dbi.commit()
confTitleIdx.clear()
confTitleIdx.initialize(dbi, ConferenceHolder().itervalues())
categNameIdx.clear()
categNameIdx.initialize(dbi, CategoryManager().itervalues())
def categoryDateIndexMigration(dbi, withRBDB, prevVersion):
"""
Replacing category date indexes.
"""
if "backupCategoryDate" not in IndexesHolder()._getIdx():
categoryDate = IndexesHolder().getIndex("categoryDate")
IndexesHolder()._getIdx()["backupCategoryDate"] = categoryDate
newIdx = CategoryDayIndex()
newIdx.buildIndex(dbi)
IndexesHolder()._getIdx()["categoryDate"] = newIdx
else:
print """categoryDateIndexMigration: new categoryDate index has """ \
"""NOT been generated because the index backup already exists.
If you still want to regenerate it, please, do it manually using """ \
"""bin/migration/CategoryDate.py"""
def reindexCategoryNameAndConferenceTitle(dbi, prevVersion):
"""
Indexing Conference Title with new WhooshTextIndex
"""
IndexesHolder().removeById('conferenceTitle')
IndexesHolder().removeById('categoryName')
confTitleIdx = IndexesHolder().getIndex('conferenceTitle')
categNameIdx = IndexesHolder().getIndex('categoryName')
dbi.commit()
iterator = (x[1] for x in console.conferenceHolderIterator(ConferenceHolder(), deepness='event'))
confTitleIdx.clear()
confTitleIdx.initialize(dbi, iterator)
categNameIdx.clear()
categNameIdx.initialize(dbi, CategoryManager().itervalues())
def event(self, query):
ch = ConferenceHolder()
index = IndexesHolder().getById("conferenceTitle")
def _iterate_objs(query_string):
query = index.search(query_string, self._orderBy)
counter = 0
# Query the DB in chunks of 1000 records per query until the limit is satisfied
for row in query.yield_per(1000):
event_id = row[0]
event = ch.getById(event_id, True)
if event is not None and event.canAccess(self._aw):
counter += 1
# Start yielding only when the counter reaches the given offset
if (self._offset is None) or (counter > self._offset):
yield event
# Stop querying the DB when the limit is satisfied
if (self._limit is not None) and (counter == self._offset + self._limit):
break
if self._orderBy in ['start', 'id', None]:
obj_list = _iterate_objs(query)
else:
obj_list = sorted(_iterate_objs(query), key=self._sortingKeys.get(self._orderBy), reverse=self._descending)
for event in obj_list:
yield {
'id': event.getId(),
'title': event.getTitle(),
'startDate': event.getStartDate(),
'hasAnyProtection': event.hasAnyProtection()
}
def countIndex(idxName):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
print len(sum(idx._words.values(), []))
DBMgr.getInstance().endRequest()
def cacheDay(dest, day):
"""
Cache a day, by calling wget in "mirror" mode
"""
dbi = DBMgr.getInstance()
dbi.startRequest()
index = {}
calIdx = IndexesHolder().getIndex('calendar')
objs = calIdx.getObjectsInDay(day)
for confId in objs:
if confId == '': continue
obj = ConferenceHolder().getById(confId)
url = str(urlHandlers.UHConferenceDisplay.getURL(obj))
savedDirs = re.match(r'http:\/\/(.*)', url).group(1).split('/')
print "Calling wget for %s..." % url
os.system(WGET_COMMAND % (confId, url,
os.path.join(dest, confId),
savedDirs[0]))
print "done!"
index[confId] = (os.path.join(confId,*savedDirs)+'.html', obj.getTitle())
dbi.endRequest(False)
return index
def category(self, idlist):
idx = IndexesHolder().getById('categoryDateAll')
filter = None
if self._room or self._location or self._eventType:
def filter(obj):
if self._eventType and obj.getType() != self._eventType:
return False
if self._location:
name = obj.getLocation() and obj.getLocation().getName()
if not name or not fnmatch.fnmatch(name.lower(),
self._location.lower()):
return False
if self._room:
name = obj.getRoom() and obj.getRoom().getName()
if not name or not fnmatch.fnmatch(name.lower(),
self._room.lower()):
return False
return True
iters = itertools.chain(
*(idx.iterateObjectsIn(catId, self._fromDT, self._toDT)
for catId in idlist))
return self._process(iters, filter)
def iter_bookings(self, idList, categ_id=None):
for vsid in idList:
if vsid not in self.ID_TO_IDX:
continue
if vsid in ['webcast', 'recording']:
idx = Catalog.getIdx('cs_booking_instance')[
self.ID_TO_IDX[vsid]]
for __, bkw in idx.iter_bookings(self._fromDT, self._toDT):
yield bkw
else:
idx = IndexesHolder().getById('collaboration')
dateFormat = '%d/%m/%Y'
tempBookings = idx.getBookings(self.ID_TO_IDX[vsid],
"conferenceStartDate",
self._orderBy, self._fromDT,
self._toDT, 'UTC', False, None,
categ_id, False,
dateFormat).getResults()
for evt, bks in tempBookings:
for bk in bks:
bk._conf = evt # Ensure all CSBookings are aware of their Conference
yield bk
def collaborationRequestIndexCreate(dbi, withRBDB, prevVersion):
"""
Creating new "All Requests" index
"""
ci = IndexesHolder().getById('collaboration')
ci.indexAll(index_names=['All Requests'], dbi=dbi)
dbi.commit()
def call(self):
cf = ConferenceHolder()
for conf in cf.getValuesToList():
csbm = Catalog.getIdx("cs_bookingmanager_conference").get(conf.getId())
csbm._bookings = {}
csbm._bookingsByType = {}
collaborationIndex = IndexesHolder().getById("collaboration")
collaborationIndex.cleanAll()
def export_categ_extra(self, aw, resultList):
ids = set((event['categoryId'] for event in resultList))
return {
'eventCategories': CategoryEventFetcher.getCategoryPath(ids, aw),
"moreFutureEvents": False if not self._toDT else
True in [IndexesHolder().getById('categoryDateAll')
.hasObjectsAfter(catId, self._toDT) for catId in self._idList]
}
def call(self):
cf = ConferenceHolder()
for conf in cf.getValuesToList():
csbm = conf.getCSBookingManager()
csbm._bookings = {}
csbm._bookingsByType = {}
collaborationIndex = IndexesHolder().getById("collaboration")
collaborationIndex.cleanAll()
def initialize(self, dbi=None):
# empty tree
self.clear()
idx = IndexesHolder().getById('collaboration')
for _, bks in idx.getBookings(self.index_name, 'conferenceStartDate', None, None, None).getResults():
for bk in bks:
self.index_booking(bk)
def buildCategoryDateIndexLtd():
""" Builds limited version of CategoryDateIndex.
Can take a long time
"""
DBMgr.getInstance().startRequest()
idx = CategoryDateIndexLtd()
idx.buildIndex()
IndexesHolder()._getIdx()["categoryDateLtd"] = idx
DBMgr.getInstance().endRequest()
def searchForId(idxName, id):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
for k, l in idx._words.iteritems():
for v in l[:]:
if v.id == id:
print "%s" % k
DBMgr.getInstance().endRequest()
def getVars(self):
vars = wcomponents.WTemplated.getVars(self)
#dictionary where the keys are names of false "indexes" for the user, and the values are IndexInformation objects
indexes = CollaborationTools.getCollaborationPluginType().getOption("pluginsPerIndex").getValue()
indexNames = set(i.getName() for i in indexes)
indexList = ['all', None, 'Vidyo', 'EVO', 'CERNMCU', 'All Videoconference', None, 'WebcastRequest', 'RecordingRequest', 'All Requests']
vars["Indexes"] = [x for x in indexList if x is None or x in indexNames]
vars["IndexInformation"] = fossilize(dict([(i.getName(), i) for i in indexes]), IIndexInformationFossil)
vars["InitialIndex"] = self._queryParams["indexName"]
vars["InitialViewBy"] = self._queryParams["viewBy"]
vars["InitialOrderBy"] = self._queryParams["orderBy"]
vars["InitialOnlyPending"] = self._queryParams["onlyPending"]
vars["InitialConferenceId"] = self._queryParams["conferenceId"]
vars["InitialCategoryId"] = self._queryParams["categoryId"]
vars["InitialSinceDate"] = self._queryParams["sinceDate"]
vars["InitialToDate"] = self._queryParams["toDate"]
vars["InitialFromDays"] = self._queryParams["fromDays"]
vars["InitialToDays"] = self._queryParams["toDays"]
vars["InitialFromTitle"] = self._queryParams["fromTitle"]
vars["InitialToTitle"] = self._queryParams["toTitle"]
vars["InitialResultsPerPage"] = self._queryParams["resultsPerPage"]
vars["InitialPage"] = self._queryParams["page"]
vars["BaseURL"] = collaborationUrlHandlers.UHAdminCollaboration.getURL()
vars["ConfCollaborationDisplay"] = collaborationUrlHandlers.UHCollaborationDisplay.getURL()
if self._queryParams["queryOnLoad"]:
ci = IndexesHolder().getById('collaboration')
tz = self._rh._getUser().getTimezone()
#####
minKey = None
maxKey = None
if self._queryParams['sinceDate']:
minKey = setAdjustedDate(datetime.strptime(self._queryParams['sinceDate'].strip(), '%Y/%m/%d'),
tz=tz)
if self._queryParams['toDate']:
maxKey = setAdjustedDate(datetime.strptime(self._queryParams['toDate'].strip(), '%Y/%m/%d'),
tz=tz)
if self._queryParams['fromTitle']:
minKey = self._queryParams['fromTitle'].strip()
if self._queryParams['toTitle']:
maxKey = self._queryParams['toTitle'].strip()
if self._queryParams['fromDays']:
try:
fromDays = int(self._queryParams['fromDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'fromDays' is not an integer"), inner = e)
midnight = nowutc().replace(hour=0, minute=0, second=0)
minKey = midnight - timedelta(days = fromDays)
if self._queryParams['toDays']:
try:
toDays = int(self._queryParams['toDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'toDays' is not an integer"), inner = e)
midnight_1 = nowutc().replace(hour=23, minute=59, second=59)
maxKey = midnight_1 + timedelta(days = toDays)
def iter_interesting_events(avatar, data):
idx = IndexesHolder().getById('categoryDateAll')
now_local = utc2server(nowutc(), False)
aw = AccessWrapper()
aw.setUser(avatar)
for event in _unique_events(
idx.iterateObjectsIn('0', now_local,
now_local + timedelta(weeks=24))):
if _is_event_interesting(avatar, event, data) and event.canAccess(aw):
yield event
def setPassportOrigin(self,value,item=None):
index = IndexesHolder().getById("registrants")
index.unindex(self)
self._passportOrigin = value
if item:
self._passportOrigin_item = item
elif self._passportOrigin_item:
self._passportOrigin_item.setValue(value)
index.index(self)
def _getAnswer(self):
idx = IndexesHolder().getById("registrants")
registrant = idx.match(self.passportID, self.passportExpire,
self.passportOrigin)
if not registrant:
return {"error": "Registrant not found"}
url = '{}/event/{}/manage/registration/users/{}/misc/0'.format(
Config.getInstance().getBaseURL(),
registrant.getConference().getId(), registrant.getId())
response = {'location': url}
return response
def inverseCheckIndex(idxName):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
if idxName == 'OAIConferenceModificationDate':
OAIConferenceModificationDate(idx).inverseCheck()
elif idxName == 'OAIPrivateConferenceModificationDate':
OAIPrivateConferenceModificationDate(idx).inverseCheck()
else:
print "No inverse checking procedures defined for %s" % idxName
sys.exit(-1)
DBMgr.getInstance().endRequest()
def _getAnswer(self):
ci = IndexesHolder().getById('collaboration')
return ci.getBookings(self._indexName,
self._viewBy,
self._orderBy,
self._minKey,
self._maxKey,
tz=self._tz,
onlyPending=self._onlyPending,
conferenceId=self._conferenceId,
categoryId=self._categoryId,
pickle=True,
dateFormat='%a %d %b %Y',
page=self._page,
resultsPerPage=self._resultsPerPage,
grouped=True)
def getEntry(idxName, entry, ids):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
if ids:
if idx._ids.has_key(entry):
print entry, idx._ids[entry]
else:
print "Key '%s' was not found" % entry
else:
if idx._words.has_key(entry):
print entry, map(prettyPrint, idx._words[entry])
else:
print "Key '%s' was not found" % entry
DBMgr.getInstance().endRequest()
def getVars( self ):
vars = wcomponents.WTemplated.getVars( self )
#dictionary where the keys are names of false "indexes" for the user, and the values are IndexInformation objects
vars["Indexes"] = CollaborationTools.getCollaborationPluginType().getOption("pluginsPerIndex").getValue()
vars["InitialIndex"] = self._queryParams["indexName"]
vars["InitialViewBy"] = self._queryParams["viewBy"]
vars["InitialOrderBy"] = self._queryParams["orderBy"]
vars["InitialOnlyPending"] = self._queryParams["onlyPending"]
vars["InitialConferenceId"] = self._queryParams["conferenceId"]
vars["InitialCategoryId"] = self._queryParams["categoryId"]
vars["InitialSinceDate"] = self._queryParams["sinceDate"]
vars["InitialToDate"] = self._queryParams["toDate"]
vars["InitialFromDays"] = self._queryParams["fromDays"]
vars["InitialToDays"] = self._queryParams["toDays"]
vars["InitialFromTitle"] = self._queryParams["fromTitle"]
vars["InitialToTitle"] = self._queryParams["toTitle"]
vars["InitialResultsPerPage"] = self._queryParams["resultsPerPage"]
vars["InitialPage"] = self._queryParams["page"]
vars["BaseURL"] = urlHandlers.UHAdminCollaboration.getURL()
if self._queryParams["queryOnLoad"]:
ci = IndexesHolder().getById('collaboration')
tz = self._rh._getUser().getTimezone()
#####
minKey = None
maxKey = None
if self._queryParams['sinceDate']:
minKey = setAdjustedDate(parseDateTime(self._queryParams['sinceDate'].strip()), tz = self._tz)
if self._queryParams['toDate']:
maxKey = setAdjustedDate(parseDateTime(self._queryParams['toDate'].strip()), tz = self._tz)
if self._queryParams['fromTitle']:
minKey = self._queryParams['fromTitle'].strip()
if self._queryParams['toTitle']:
maxKey = self._queryParams['toTitle'].strip()
if self._queryParams['fromDays']:
try:
fromDays = int(self._queryParams['fromDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'fromDays' is not an integer"), inner = e)
minKey = nowutc() - timedelta(days = fromDays)
if self._queryParams['toDays']:
try:
toDays = int(self._queryParams['toDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'toDays' is not an integer"), inner = e)
maxKey = nowutc() + timedelta(days = toDays)
def switchIndex():
if IndexesHolder()._getIdx().has_key(
"backupCategoryDate") and IndexesHolder()._getIdx().has_key(
"categoryDate"):
tmp = IndexesHolder()._getIdx()["backupCategoryDate"]
IndexesHolder()._getIdx()["backupCategoryDate"] = IndexesHolder(
).getIndex("categoryDate")
IndexesHolder()._getIdx()["categoryDate"] = tmp
print "Index was switched"
else:
print "Cannot switch indexes."
def fixIndexesEncoding(dbi, withRBDB, prevVersion):
"""
Fix indexes encoding. They may be in unicode and they have to be encoded in utf-8
"""
INDEXES = ["name", "surName", "organisation"]
ih = IndexesHolder()
for idx_name in INDEXES:
idx = ih.getById(idx_name)
words = idx._words
for key in list(words):
newKey = fix_broken_string(key)
values = words[key]
del words[key]
words[newKey] = values
idx.setIndex(words)
dbi.commit()
def listIndex(idxName, ids):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
for k, v in idx._words.iteritems():
if ids:
f = lambda x: str(x.getId())
else:
f = prettyPrint
print k, map(f, v)
DBMgr.getInstance().endRequest()
## def fixOrphans(idxName):
## DBMgr.getInstance().startRequest()
## idx = IndexesHolder().getIndex(idxName)
## for k,l in idx._words.iteritems():
## for v in l[:]:
## if type(v) != MaKaC.conference.DeletedObject:
## parent = getParent(v)
## orphan = False
## if parent == None:
## print "[1st deg ORPHAN] %s" % prettyPrint(v)
## orphan = True
## elif hasattr(parent, 'getOwnerList') and parent.getOwnerList() == []:
## print "[2nd deg ORPHAN] %s %s" % (prettyPrint(v), parent)
## orphan = True
## if orphan:
## words = idx._words
## l = words[k]
## l.remove(v)
## words[k] = l
## idx.setIndex(words)
## ids = idx._ids
## l = words[k]
## l.remove(v.id)
## ids[k] = l
DBMgr.getInstance().endRequest()
