def cacheDay(dest, day):
"""
Cache a day, by calling wget in "mirror" mode
"""
dbi = DBMgr.getInstance()
dbi.startRequest()
index = {}
calIdx = IndexesHolder().getIndex('calendar')
objs = calIdx.getObjectsInDay(day)
for confId in objs:
if confId == '': continue
obj = ConferenceHolder().getById(confId)
url = str(urlHandlers.UHConferenceDisplay.getURL(obj))
savedDirs = re.match(r'http:\/\/(.*)', url).group(1).split('/')
print "Calling wget for %s..." % url
os.system(WGET_COMMAND % (confId, url,
os.path.join(dest, confId),
savedDirs[0]))
print "done!"
index[confId] = (os.path.join(confId,*savedDirs)+'.html', obj.getTitle())
dbi.endRequest(False)
return index
def iter_bookings(self, idList, categ_id=None):
for vsid in idList:
if vsid in ["webcast", "recording"]:
idx = Catalog.getIdx("cs_booking_instance")[self.ID_TO_IDX[vsid]]
for __, bkw in idx.iter_bookings(self._fromDT, self._toDT):
yield bkw
else:
idx = IndexesHolder().getById("collaboration")
dateFormat = "%d/%m/%Y"
tempBookings = idx.getBookings(
self.ID_TO_IDX[vsid],
"conferenceStartDate",
self._orderBy,
self._fromDT,
self._toDT,
"UTC",
False,
None,
categ_id,
False,
dateFormat,
).getResults()
for evt, bks in tempBookings:
for bk in bks:
bk._conf = evt # Ensure all CSBookings are aware of their Conference
yield bk
def collaborationRequestIndexCreate(dbi, withRBDB, prevVersion):
"""
Creating new "All Requests" index
"""
ci = IndexesHolder().getById('collaboration')
ci.indexAll(index_names=['All Requests'], dbi=dbi)
dbi.commit()
def video(self, idList, alarm = None):
idx = IndexesHolder().getById('collaboration');
bookings = []
dateFormat = '%d/%m/%Y'
self._alarm = alarm
for vsid in idList:
tempBookings = idx.getBookings(self.ID_TO_IDX[vsid], "conferenceStartDate",
self._orderBy, self._fromDT, self._toDT,
'UTC', False, None, None, False, dateFormat)
bookings.extend(tempBookings.getResults())
def _iter_bookings(objs):
for obj in objs:
for bk in obj[1]:
bk._conf = obj[0] # Ensure all CSBookings are aware of their Conference
""" This is for plugins whose structure include 'talkSelected',
examples of which in CERN Indico being WebcastRequest and
RecordingRequest.
"""
if bk.hasTalkSelection():
ts = bk.getTalkSelectionList()
contributions = []
if ts is None: # No individual talks, therefore an event for every contribution
contributions = bk._conf.getContributionList()
else:
for contribId in ts:
tempContrib = bk._conf.getContributionById(contribId)
contributions.append(tempContrib)
if len(contributions) == 0 or self._detail == "event": # If we are here, no contributions but a request exists.
bk.setStartDate(bk._conf.getStartDate())
bk.setEndDate(bk._conf.getEndDate())
yield bk
else: # Contributions is the list of all to be exported now
contributions = filter(lambda c: c.isScheduled(), contributions)
for contrib in contributions:
# Wrap the CSBooking object for export
bkw = CSBookingContributionWrapper(bk, contrib)
bkw.setStartDate(contrib.getStartDate())
bkw.setEndDate(contrib.getEndDate())
yield bkw
else:
yield bk
""" Simple filter, as this method can return None for Pending and True
for accepted, both are valid booking statuses for exporting.
"""
def _filter(obj):
return obj.getAcceptRejectStatus() is not False
iface = self.DETAIL_INTERFACES.get('all')
for booking in self._process(_iter_bookings(bookings), _filter, iface):
yield booking
def call(self):
cf = ConferenceHolder()
for conf in cf.getValuesToList():
csbm = conf.getCSBookingManager()
csbm._bookings = {}
csbm._bookingsByType = {}
collaborationIndex = IndexesHolder().getById("collaboration")
collaborationIndex.cleanAll()
def call(self):
cf = ConferenceHolder()
for conf in cf.getValuesToList():
csbm = Catalog.getIdx("cs_bookingmanager_conference").get(conf.getId())
csbm._bookings = {}
csbm._bookingsByType = {}
collaborationIndex = IndexesHolder().getById("collaboration")
collaborationIndex.cleanAll()
def setUp(self):
DBMgr.getInstance().startRequest()
self.oldIndex = IndexesHolder()._getIdx()["categoryDateLtd"]
self.newIndex = IndexesHolder()._getIdx()["categoryDate"]
self.startDate = datetime(2010,5,13, 10, 0, 0, tzinfo=timezone('UTC'))
self.endDate = datetime(2010,5,14, 14, 0, 0, tzinfo=timezone('UTC'))
self.ch = ConferenceHolder()
self.categId = '0'
def iter_interesting_events(avatar, data):
idx = IndexesHolder().getById('categoryDateAll')
now_local = utc2server(nowutc(), False)
aw = AccessWrapper()
aw.setUser(avatar)
for event in _unique_events(idx.iterateObjectsIn('0', now_local, now_local + timedelta(weeks=24))):
if _is_event_interesting(avatar, event, data) and event.canAccess(aw):
yield event
def initialize(self, dbi=None):
# empty tree
self.clear()
idx = IndexesHolder().getById('collaboration')
for conf, bks in idx.getBookings(self.index_name, 'conferenceStartDate', None, None, None).getResults():
for bk in bks:
self.index_booking(bk)
def initialize(self, dbi=None):
# empty tree
self.clear()
idx = IndexesHolder().getById('collaboration')
for _, bks in idx.getBookings(self.index_name, 'conferenceStartDate', None, None, None).getResults():
for bk in bks:
self.index_booking(bk)
def indexConferenceTitle(dbi, withRBDB, prevVersion):
"""
Indexing Conference Title
"""
ch = ConferenceHolder()
nameIdx = IndexesHolder().getIndex('conferenceTitle')
i = 0
for (__, conf) in console.conferenceHolderIterator(ch, deepness='event'):
nameIdx.index(conf.getId(), conf.getTitle().decode('utf-8'))
i += 1
if i % 10000 == 0:
dbi.commit()
def export_event(self, aw):
ch = ConferenceHolder()
index = IndexesHolder().getIndex('conferenceTitle')
try:
query = ' AND '.join(map(lambda y: "*%s*" % y, filter(lambda x: len(x) > 0, self._search.split(' '))))
results = index.search(query)
except parsetree.ParseError:
results = []
d = []
for id, v in results:
event = ch.getById(id)
if event.canAccess(aw):
d.append({'id': id, 'title': event.getTitle(), 'startDate': event.getStartDate(), 'hasAnyProtection': event.hasAnyProtection()})
return d
def reindexCategoryNameAndConferenceTitle(dbi, withRBDB, prevVersion):
"""
Indexing Conference Title with new WhooshTextIndex
"""
IndexesHolder().removeById('conferenceTitle')
IndexesHolder().removeById('categoryName')
confTitleIdx = IndexesHolder().getIndex('conferenceTitle')
categNameIdx = IndexesHolder().getIndex('categoryName')
dbi.commit()
confTitleIdx.clear()
confTitleIdx.initialize(dbi, ConferenceHolder().itervalues())
categNameIdx.clear()
categNameIdx.initialize(dbi, CategoryManager().itervalues())
def _getAnswer(self):
ci = IndexesHolder().getById('collaboration')
return ci.getBookings(self._indexName,
self._viewBy,
self._orderBy,
self._minKey,
self._maxKey,
tz = self._tz,
onlyPending = self._onlyPending,
conferenceId = self._conferenceId,
categoryId = self._categoryId,
pickle = True,
dateFormat='%a %d %b %Y',
page = self._page,
resultsPerPage = self._resultsPerPage,
grouped=True)
def event(self, query):
ch = ConferenceHolder()
index = IndexesHolder().getById("conferenceTitle")
def _iterate_objs(query_string):
query = index.search(query_string, self._orderBy)
counter = 0
# Query the DB in chunks of 1000 records per query until the limit is satisfied
for row in query.yield_per(1000):
event_id = row[0]
event = ch.getById(event_id, True)
if event is not None and event.canAccess(self._aw):
counter += 1
# Start yielding only when the counter reaches the given offset
if (self._offset is None) or (counter > self._offset):
yield event
# Stop querying the DB when the limit is satisfied
if (self._limit is not None) and (counter == self._offset + self._limit):
break
if self._orderBy in ['start', 'id', None]:
obj_list = _iterate_objs(query)
else:
obj_list = sorted(_iterate_objs(query), key=self._sortingKeys.get(self._orderBy), reverse=self._descending)
for event in obj_list:
yield {
'id': event.getId(),
'title': event.getTitle(),
'startDate': event.getStartDate(),
'hasAnyProtection': event.hasAnyProtection()
}
def reindexCategoryNameAndConferenceTitle(dbi, prevVersion):
"""
Indexing Conference Title with new WhooshTextIndex
"""
IndexesHolder().removeById('conferenceTitle')
IndexesHolder().removeById('categoryName')
confTitleIdx = IndexesHolder().getIndex('conferenceTitle')
categNameIdx = IndexesHolder().getIndex('categoryName')
dbi.commit()
iterator = (x[1] for x in console.conferenceHolderIterator(ConferenceHolder(), deepness='event'))
confTitleIdx.clear()
confTitleIdx.initialize(dbi, iterator)
categNameIdx.clear()
categNameIdx.initialize(dbi, CategoryManager().itervalues())
def countIndex(idxName):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
print len(sum(idx._words.values(), []))
DBMgr.getInstance().endRequest()
def _getAnswer(self):
idx = IndexesHolder().getById("registrants")
registrant = idx.match(self.passportID,self.passportExpire,self.passportOrigin)
if not registrant:
return {"error":"Registrant not found"}
url = '{}/event/{}/manage/registration/users/{}/misc/0'.format(
Config.getInstance().getBaseURL(),
registrant.getConference().getId(),
registrant.getId()
)
response = {
'location': url
}
return response
def export_categ_extra(self, aw, resultList):
ids = set((event['categoryId'] for event in resultList))
return {
'eventCategories': CategoryEventFetcher.getCategoryPath(ids, aw),
"moreFutureEvents": False if not self._toDT else
True in [IndexesHolder().getById('categoryDateAll')
.hasObjectsAfter(catId, self._toDT) for catId in self._idList]
}
def searchForId(idxName, id):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
for k, l in idx._words.iteritems():
for v in l[:]:
if v.id == id:
print "%s" % k
DBMgr.getInstance().endRequest()
def fixIndexesEncoding(dbi, withRBDB, prevVersion):
"""
Fix indexes encoding. They may be in unicode and they have to be encoded in utf-8
"""
INDEXES = ["name", "surName", "organisation"]
ih = IndexesHolder()
for idx_name in INDEXES:
idx = ih.getById(idx_name)
words = idx._words
for key in list(words):
newKey = fix_broken_string(key)
values = words[key]
del words[key]
words[newKey] = values
idx.setIndex(words)
dbi.commit()
def fix_indexes():
dbi = DBMgr.getInstance()
dbi.startRequest()
ih = IndexesHolder()
for idx_name in INDEXES:
idx = ih.getById(idx_name)
words = idx._words
for key in words.iterkeys():
newKey = fix_broken_string(key)
values = words[key]
del words[key]
words[newKey] = values
idx.setIndex(words)
dbi.commit()
dbi.endRequest()
def fixIndexesEncoding(dbi, withRBDB, prevVersion):
"""
Fix indexes encoding. They may be in unicode and they has to be encoded in utf-8
"""
INDEXES = ["name", "surName", "organisation"]
ih = IndexesHolder()
for idx_name in INDEXES:
idx = ih.getById(idx_name)
words = idx._words
for key in words.iterkeys():
newKey = fix_broken_string(key)
values = words[key]
del words[key]
words[newKey] = values
idx.setIndex(words)
dbi.commit()
def getVars(self):
vars = wcomponents.WTemplated.getVars(self)
#dictionary where the keys are names of false "indexes" for the user, and the values are IndexInformation objects
indexes = CollaborationTools.getCollaborationPluginType().getOption("pluginsPerIndex").getValue()
indexNames = set(i.getName() for i in indexes)
indexList = ['all', None, 'Vidyo', 'EVO', 'CERNMCU', 'All Videoconference', None, 'WebcastRequest', 'RecordingRequest', 'All Requests']
vars["Indexes"] = [x for x in indexList if x is None or x in indexNames]
vars["IndexInformation"] = fossilize(dict([(i.getName(), i) for i in indexes]), IIndexInformationFossil)
vars["InitialIndex"] = self._queryParams["indexName"]
vars["InitialViewBy"] = self._queryParams["viewBy"]
vars["InitialOrderBy"] = self._queryParams["orderBy"]
vars["InitialOnlyPending"] = self._queryParams["onlyPending"]
vars["InitialConferenceId"] = self._queryParams["conferenceId"]
vars["InitialCategoryId"] = self._queryParams["categoryId"]
vars["InitialSinceDate"] = self._queryParams["sinceDate"]
vars["InitialToDate"] = self._queryParams["toDate"]
vars["InitialFromDays"] = self._queryParams["fromDays"]
vars["InitialToDays"] = self._queryParams["toDays"]
vars["InitialFromTitle"] = self._queryParams["fromTitle"]
vars["InitialToTitle"] = self._queryParams["toTitle"]
vars["InitialResultsPerPage"] = self._queryParams["resultsPerPage"]
vars["InitialPage"] = self._queryParams["page"]
vars["BaseURL"] = collaborationUrlHandlers.UHAdminCollaboration.getURL()
vars["ConfCollaborationDisplay"] = collaborationUrlHandlers.UHCollaborationDisplay.getURL()
if self._queryParams["queryOnLoad"]:
ci = IndexesHolder().getById('collaboration')
tz = self._rh._getUser().getTimezone()
#####
minKey = None
maxKey = None
if self._queryParams['sinceDate']:
minKey = setAdjustedDate(datetime.strptime(self._queryParams['sinceDate'].strip(), '%Y/%m/%d'),
tz=tz)
if self._queryParams['toDate']:
maxKey = setAdjustedDate(datetime.strptime(self._queryParams['toDate'].strip(), '%Y/%m/%d'),
tz=tz)
if self._queryParams['fromTitle']:
minKey = self._queryParams['fromTitle'].strip()
if self._queryParams['toTitle']:
maxKey = self._queryParams['toTitle'].strip()
if self._queryParams['fromDays']:
try:
fromDays = int(self._queryParams['fromDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'fromDays' is not an integer"), inner = e)
midnight = nowutc().replace(hour=0, minute=0, second=0)
minKey = midnight - timedelta(days = fromDays)
if self._queryParams['toDays']:
try:
toDays = int(self._queryParams['toDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'toDays' is not an integer"), inner = e)
midnight_1 = nowutc().replace(hour=23, minute=59, second=59)
maxKey = midnight_1 + timedelta(days = toDays)
def category(self, idlist):
idx = IndexesHolder().getById('categoryDateAll')
filter = None
if self._room or self._location or self._eventType:
def filter(obj):
if self._eventType and obj.getType() != self._eventType:
return False
if self._location:
name = obj.getLocation() and obj.getLocation().getName()
if not name or not fnmatch.fnmatch(name.lower(), self._location.lower()):
return False
if self._room:
name = obj.getRoom() and obj.getRoom().getName()
if not name or not fnmatch.fnmatch(name.lower(), self._room.lower()):
return False
return True
iters = itertools.chain(*(idx.iterateObjectsIn(catId, self._fromDT, self._toDT) for catId in idlist))
return self._process(iters, filter)
def iter_bookings(self, idList, categ_id=None):
for vsid in idList:
if vsid not in self.ID_TO_IDX:
continue
if vsid in ['webcast', 'recording']:
idx = Catalog.getIdx('cs_booking_instance')[self.ID_TO_IDX[vsid]]
for __, bkw in idx.iter_bookings(self._fromDT, self._toDT):
yield bkw
else:
idx = IndexesHolder().getById('collaboration');
dateFormat = '%d/%m/%Y'
tempBookings = idx.getBookings(self.ID_TO_IDX[vsid], "conferenceStartDate",
self._orderBy, self._fromDT, self._toDT,
'UTC', False, None, categ_id, False, dateFormat).getResults()
for evt, bks in tempBookings:
for bk in bks:
bk._conf = evt # Ensure all CSBookings are aware of their Conference
yield bk
def _process( self ):
from datetime import datetime,timedelta
from MaKaC.common.indexes import IndexesHolder
self._responseUtil.content_type = 'text/xml'
XG = xmlGen.XMLGen()
XG.openTag("response")
now = startdt = enddt = datetime.now()
today = startdt.date()
startdt.replace( hour = 0, minute = 0)
enddt.replace( hour = 23, minute = 59)
calIdx = IndexesHolder().getById("calendar")
nbEvtsToday = len(calIdx.getObjectsInDay(now))
nbOngoingEvts = len(calIdx.getObjectsIn(now,now))
self._createIndicator(XG, "nbEventsToday", "total number of events for today", nbEvtsToday)
self._createIndicator(XG, "nbOngoingEvents", "total number of ongoing events", nbOngoingEvts)
XG.closeTag("response")
return XG.getXml()
def _process( self ):
from datetime import datetime,timedelta
from MaKaC.common.indexes import IndexesHolder
self._req.content_type = "text/xml"
XG = xmlGen.XMLGen()
XG.openTag("response")
now = startdt = enddt = datetime.now()
today = startdt.date()
startdt.replace( hour = 0, minute = 0)
enddt.replace( hour = 23, minute = 59)
calIdx = IndexesHolder().getById("calendar")
nbEvtsToday = len(calIdx.getObjectsInDay(now))
nbOngoingEvts = len(calIdx.getObjectsIn(now,now))
self._createIndicator(XG, "nbEventsToday", "total number of events for today", nbEvtsToday)
self._createIndicator(XG, "nbOngoingEvents", "total number of ongoing events", nbOngoingEvts)
XG.closeTag("response")
return XG.getXml()
def inverseCheckIndex(idxName):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
if idxName == 'OAIConferenceModificationDate':
OAIConferenceModificationDate(idx).inverseCheck()
elif idxName == 'OAIPrivateConferenceModificationDate':
OAIPrivateConferenceModificationDate(idx).inverseCheck()
else:
print "No inverse checking procedures defined for %s" % idxName
sys.exit(-1)
DBMgr.getInstance().endRequest()
def getEntry(idxName, entry, ids):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
if ids:
if idx._ids.has_key(entry):
print entry, idx._ids[entry]
else:
print "Key '%s' was not found" % entry
else:
if idx._words.has_key(entry):
print entry, map(prettyPrint, idx._words[entry])
else:
print "Key '%s' was not found" % entry
DBMgr.getInstance().endRequest()
def getVars( self ):
vars = wcomponents.WTemplated.getVars( self )
#dictionary where the keys are names of false "indexes" for the user, and the values are IndexInformation objects
vars["Indexes"] = CollaborationTools.getCollaborationPluginType().getOption("pluginsPerIndex").getValue()
vars["InitialIndex"] = self._queryParams["indexName"]
vars["InitialViewBy"] = self._queryParams["viewBy"]
vars["InitialOrderBy"] = self._queryParams["orderBy"]
vars["InitialOnlyPending"] = self._queryParams["onlyPending"]
vars["InitialConferenceId"] = self._queryParams["conferenceId"]
vars["InitialCategoryId"] = self._queryParams["categoryId"]
vars["InitialSinceDate"] = self._queryParams["sinceDate"]
vars["InitialToDate"] = self._queryParams["toDate"]
vars["InitialFromDays"] = self._queryParams["fromDays"]
vars["InitialToDays"] = self._queryParams["toDays"]
vars["InitialFromTitle"] = self._queryParams["fromTitle"]
vars["InitialToTitle"] = self._queryParams["toTitle"]
vars["InitialResultsPerPage"] = self._queryParams["resultsPerPage"]
vars["InitialPage"] = self._queryParams["page"]
vars["BaseURL"] = urlHandlers.UHAdminCollaboration.getURL()
if self._queryParams["queryOnLoad"]:
ci = IndexesHolder().getById('collaboration')
tz = self._rh._getUser().getTimezone()
#####
minKey = None
maxKey = None
if self._queryParams['sinceDate']:
minKey = setAdjustedDate(parseDateTime(self._queryParams['sinceDate'].strip()), tz = self._tz)
if self._queryParams['toDate']:
maxKey = setAdjustedDate(parseDateTime(self._queryParams['toDate'].strip()), tz = self._tz)
if self._queryParams['fromTitle']:
minKey = self._queryParams['fromTitle'].strip()
if self._queryParams['toTitle']:
maxKey = self._queryParams['toTitle'].strip()
if self._queryParams['fromDays']:
try:
fromDays = int(self._queryParams['fromDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'fromDays' is not an integer"), inner = e)
minKey = nowutc() - timedelta(days = fromDays)
if self._queryParams['toDays']:
try:
toDays = int(self._queryParams['toDays'])
except ValueError, e:
raise CollaborationException(_("Parameter 'toDays' is not an integer"), inner = e)
maxKey = nowutc() + timedelta(days = toDays)
def setPassportOrigin(self,value,item=None):
index = IndexesHolder().getById("registrants")
index.unindex(self)
self._passportOrigin = value
if item:
self._passportOrigin_item = item
elif self._passportOrigin_item:
self._passportOrigin_item.setValue(value)
index.index(self)
def listIndex(idxName, ids):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
for k, v in idx._words.iteritems():
if ids:
f = lambda x: str(x.getId())
else:
f = prettyPrint
print k, map(f, v)
DBMgr.getInstance().endRequest()
## def fixOrphans(idxName):
## DBMgr.getInstance().startRequest()
## idx = IndexesHolder().getIndex(idxName)
## for k,l in idx._words.iteritems():
## for v in l[:]:
## if type(v) != MaKaC.conference.DeletedObject:
## parent = getParent(v)
## orphan = False
## if parent == None:
## print "[1st deg ORPHAN] %s" % prettyPrint(v)
## orphan = True
## elif hasattr(parent, 'getOwnerList') and parent.getOwnerList() == []:
## print "[2nd deg ORPHAN] %s %s" % (prettyPrint(v), parent)
## orphan = True
## if orphan:
## words = idx._words
## l = words[k]
## l.remove(v)
## words[k] = l
## idx.setIndex(words)
## ids = idx._ids
## l = words[k]
## l.remove(v.id)
## ids[k] = l
DBMgr.getInstance().endRequest()
def checkIndex(idxName, dfrom, fix, showProgress):
DBMgr.getInstance().startRequest()
idx = IndexesHolder().getIndex(idxName)
if idxName == 'OAIConferenceModificationDate':
OAIConferenceModificationDate(idx).check(dfrom, fix, showProgress)
elif idxName == 'OAIPrivateConferenceModificationDate':
OAIPrivateConferenceModificationDate(idx).check(
dfrom, fix, showProgress)
elif idxName == 'OAIContributionModificationDate':
OAIContributionModificationDate(idx).check(dfrom, fix, showProgress)
elif idxName == 'OAIPrivateContributionModificationDate':
OAIPrivateContributionModificationDate(idx).check(
dfrom, fix, showProgress)
else:
print "No checking procedures defined for %s" % idxName
sys.exit(-1)
DBMgr.getInstance().endRequest()
def category_events(self, catIds):
idx = IndexesHolder().getById('categoryDateAll')
iters = itertools.chain(*(idx.iterateObjectsIn(catId, self._fromDT, self._toDT) for catId in catIds))
return self._process(iters)
from MaKaC.common import DBMgr
from MaKaC.common.indexes import IndexesHolder
from indico.core.index import Catalog
if __name__ == '__main__':
dbi = DBMgr.getInstance()
dbi.startRequest()
for idx_name in ['categ_conf_sd']:
idx = Catalog.getIdx(idx_name)
for problem in idx._check(dbi=dbi):
print "[%s] %s" % (idx_name, problem)
for idx_name in ['category', 'calendar', 'categoryDate']:
idx = IndexesHolder().getIndex(idx_name)
for problem in idx._check(dbi=dbi):
print "[%s] %s" % (idx_name, problem)
dbi.endRequest()
class TestCategoryDayIndex(unittest.TestCase):
def setUp(self):
DBMgr.getInstance().startRequest()
self.oldIndex = IndexesHolder()._getIdx()["categoryDateLtd"]
self.newIndex = IndexesHolder()._getIdx()["categoryDate"]
self.startDate = datetime(2010,5,13, 10, 0, 0, tzinfo=timezone('UTC'))
self.endDate = datetime(2010,5,14, 14, 0, 0, tzinfo=timezone('UTC'))
self.ch = ConferenceHolder()
self.categId = '0'
def tearDownModule(self):
DBMgr.getInstance().abort()
DBMgr.getInstance().endRequest()
def testGetObjectsStartingInDay(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsStartingInDay(self.startDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsStartingInDay(self.startDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsStartingIn(self):
newRes = self.newIndex.getObjectsStartingIn(self.categId, self.startDate, self.endDate)
oldTmp = self.oldIndex.getObjectsStartingIn(self.categId, self.startDate, self.endDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsStartingAfterTest(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsStartingAfter(self.startDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsStartingAfter(self.startDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsEndingAfterTest(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsEndingAfter(self.startDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsEndingAfter(self.startDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsEndingInDay(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsEndingInDay(self.startDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsEndingInDay(self.startDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsEndingIn(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsEndingIn(self.startDate,self.endDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsEndingIn(self.startDate,self.endDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsInDay(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsInDay(self.startDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsInDay(self.startDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def testGetObjectsIn(self):
newRes = self.newIndex._idxCategItem[self.categId].getObjectsIn(self.startDate,self.endDate)
oldTmp = self.oldIndex._idxCategItem[self.categId].getObjectsIn(self.startDate,self.endDate)
oldRes = set()
for res in oldTmp:
oldRes.add(self.ch.getById(res))
assert(oldRes == newRes)
def reindex(fix=False,fromDate=None):
""" Recreate deleted obj indices, from the DOH """
dbi = DBMgr.getInstance()
dbi.startRequest()
pubConfIdx = IndexesHolder().getIndex('OAIDeletedConferenceModificationDate')
prvConfIdx = IndexesHolder().getIndex('OAIDeletedPrivateConferenceModificationDate')
pubContIdx = IndexesHolder().getIndex('OAIDeletedContributionModificationDate')
prvContIdx = IndexesHolder().getIndex('OAIDeletedPrivateContributionModificationDate')
doh = DeletedObjectHolder()
pubConfIdx.initIndex()
pubContIdx.initIndex()
prvConfIdx.initIndex()
prvContIdx.initIndex()
fromDateParsed_tz = datetime.datetime(*time.strptime(fromDate,'%Y-%m-%d')[:6],**{'tzinfo':timezone('UTC')})
fromDateParsed_naive = datetime.datetime(*time.strptime(fromDate,'%Y-%m-%d')[:6])
for obj in doh.getList():
if fromDate:
if obj.getOAIModificationDate().tzinfo:
fromDateParsed = fromDateParsed_tz
else:
fromDateParsed = fromDateParsed_naive
if obj.getOAIModificationDate() < fromDateParsed:
continue
if not hasattr(obj,'protected'):
print "NO DATA FOR %s (%s)" % (obj.getId(), obj.getOAIModificationDate())
continue
print "indexing %s (%s)" % (prettyPrint(obj), obj.getOAIModificationDate())
if obj._objClass == Conference:
if obj.protected:
prvConfIdx.indexConference(obj)
else:
pubConfIdx.indexConference(obj)
elif obj._objClass == Contribution or obj._objClass == AcceptedContribution or obj._objClass == SubContribution:
if obj.protected:
prvContIdx.indexContribution(obj)
else:
pubContIdx.indexContribution(obj)
dbi.commit()
dbi.endRequest()
def findZombies(fix=False,fromDate=None):
dbi = DBMgr.getInstance()
dbi.startRequest()
pubConfIdx = IndexesHolder().getIndex('OAIDeletedConferenceModificationDate')
prvConfIdx = IndexesHolder().getIndex('OAIDeletedPrivateConferenceModificationDate')
pubContIdx = IndexesHolder().getIndex('OAIDeletedContributionModificationDate')
prvContIdx = IndexesHolder().getIndex('OAIDeletedPrivateContributionModificationDate')
zombies = []
doh = DeletedObjectHolder()
for obj in doh.getList():
isZombie = False
rec = None
if obj._objClass == Conference:
try:
rec = ConferenceHolder().getById(obj.getId())
isZombie = True
except:
continue
elif obj._objClass == Contribution or obj._objClass == AcceptedContribution:
try:
conference = ConferenceHolder().getById(obj.confId)
except:
continue
rec = conference.getContributionById(obj.contribId)
isZombie = (rec != None)
elif obj._objClass == SubContribution:
try:
conference = ConferenceHolder().getById(obj.confId)
except:
continue
contrib = conference.getContributionById(obj.contribId)
if not contrib:
continue
rec = contrib.getSubContributionById(obj.subContribId)
isZombie = (rec != None)
if isZombie:
print "-- ZOMBIE %s" % prettyPrint(rec)
zombies.append(obj)
if fix:
for z in zombies:
dbi.sync()
pubConfIdx.unindexElement(z)
prvConfIdx.unindexElement(z)
pubContIdx.unindexElement(z)
prvContIdx.unindexElement(z)
id = z.getId()
doh.remove(z)
dbi.commit()
print "-- FIXED %s " % id
dbi.endRequest()
print "\n Total of %s zombie records found" % len(zombies)
return zombies
def call(self):
collaborationIndex = IndexesHolder().getById("collaboration")
collaborationIndex.reindexAll()
def _get_category_score(user, categ, attended_events, debug=False):
# avoid stale SQLAlchemy object
if debug:
print repr(categ)
idx = IndexesHolder().getById("categoryDateAll")
attended_events_set = set(attended_events)
# We care about events in the whole timespan where the user attended some events.
# However, this might result in some missed events e.g. if the user was not working for
# a year and then returned. So we throw away old blocks (or rather adjust the start time
# to the start time of the newest block)
first_event_date = attended_events[0].getStartDate().replace(hour=0, minute=0)
last_event_date = attended_events[-1].getStartDate().replace(hour=0, minute=0) + timedelta(days=1)
blocks = _get_blocks(
_unique_events(idx.iterateObjectsIn(categ.getId(), first_event_date, last_event_date)), attended_events_set
)
for a, b in _window(blocks):
# More than 3 months between blocks? Ignore the old block!
if b[0].getStartDate() - a[-1].getStartDate() > timedelta(weeks=12):
first_event_date = b[0].getStartDate().replace(hour=0, minute=0)
# Favorite categories get a higher base score
favorite = categ in user.favorite_categories
score = 1 if favorite else 0
if debug:
print "{0:+.3f} - initial".format(score)
# Attendance percentage goes to the score directly. If the attendance is high chances are good that the user
# is either very interested in whatever goes on in the category or it's something he has to attend regularily.
total = sum(1 for _ in _unique_events(idx.iterateObjectsIn(categ.getId(), first_event_date, last_event_date)))
attended_block_event_count = sum(1 for e in attended_events_set if e.getStartDate() >= first_event_date)
score += attended_block_event_count / total
if debug:
print "{0:+.3f} - attendance".format(score)
# If there are lots/few unattended events after the last attended one we also update the score with that
total_after = sum(
1 for _ in _unique_events(idx.iterateObjectsIn(categ.getId(), last_event_date + timedelta(days=1), None))
)
if total_after < total * 0.05:
score += 0.25
elif total_after > total * 0.25:
score -= 0.5
if debug:
print "{0:+.3f} - unattended new events".format(score)
# Lower the score based on how long ago the last attended event was if there are no future events
# We start applying this modifier only if the event has been more than 40 days in the past to avoid
# it from happening in case of monthly events that are not created early enough.
days_since_last_event = (date.today() - last_event_date.date()).days
if days_since_last_event > 40:
score -= 0.025 * days_since_last_event
if debug:
print "{0:+.3f} - days since last event".format(score)
# For events in the future however we raise the score
now_local = utc2server(nowutc(), False)
attending_future = [
e
for e in _unique_events(idx.iterateObjectsIn(categ.getId(), now_local, last_event_date))
if e in attended_events_set
]
if attending_future:
score += 0.25 * len(attending_future)
if debug:
print "{0:+.3f} - future event count".format(score)
days_to_future_event = (attending_future[0].getStartDate().date() - date.today()).days
score += max(0.1, -(max(0, days_to_future_event - 2) / 4) ** (1 / 3) + 2.5)
if debug:
print "{0:+.3f} - days to next future event".format(score)
return score