def next(self):
start = time.time()
searchers = []
postingLists = []
try:
# get a searcher and posting list for each index
for index in self._indices:
# we create a copy of the original query, which can possibly be optimized
# with index-specific knowledge.
query = copy.deepcopy(self._query)
# get the posting list to iterate through
searcher = yield index.newSearcher()
if not ISearcher.providedBy(searcher):
raise TypeError("searcher does not implement ISearcher")
query = yield query.optimizeMatcher(searcher)
logger.debug("optimized query for index '%s': %s" %
(index.name, str(query)))
# if the query optimized out entirely, then skip to the next index
if query == None:
yield searcher.close()
continue
postingList = yield query.iterMatches(searcher, self._startId,
self._endId)
if not IPostingList.providedBy(postingList):
raise TypeError(
"posting list does not implement IPostingList")
searchers.append(searcher)
postingLists.append(postingList)
if len(postingLists) == 0:
raise StopIteration()
# loop forever until we reach the search limit, we exhaust all of our
# posting lists, or we encounter an exception
currPostings = [(None, None, None)
for i in range(len(postingLists))]
smallestList = 0
lastId = None
compar = (lambda x, y: cmp(y, x)) if self._reverse else cmp
while True:
# if we have reached our limit
if len(self.events) == self._limit:
self.runtime = time.time() - start
raise StopIteration()
# check each child iter for the lowest evid
for currList in range(len(postingLists)):
if currList == None:
smallestList = 0
# if the postingList is None, then its closed
if postingLists[currList] == None:
# FIXME: close the posting list and searcher
continue
# if current posting for this posting list was not consumed
if currPostings[currList] != (None, None, None):
continue
# otherwise get the next posting from the posting list
currPostings[currList] = yield postingLists[
currList].nextPosting()
# if the next posting for this posting list is (None,None,None),
# then we are done with this posting list
if currPostings[currList] == (None, None, None):
postingList = postingLists[currList]
yield postingList.close()
postingLists[currList] = None
continue
# if the evid equals the last evid returned, then ignore it
if lastId != None and currPostings[currList][0] == lastId:
currPostings[currList] = (None, None, None)
continue
# we don't compare the first evid with itself
if currList == 0:
continue
# if the evid is the current smallest evid, then remember it
if (currPostings[smallestList] == (None, None, None)
or compar(currPostings[currList][0],
currPostings[smallestList][0]) < 0):
smallestList = currList
# get the next posting
currList = None
evid, _, store = currPostings[smallestList]
# stop iterating if there are no more results
if evid == None:
self.runtime = time.time() - start
raise StopIteration()
# remember the last evid
lastId = evid
# forget the evid so we don't return it again
currPostings[smallestList] = (None, None, None)
# retrieve the event
if not IEventStore.providedBy(store):
raise TypeError("store does not implement IEventStore")
event = yield store.getEvent(evid)
defaultfield, defaultvalue, fields = event
if defaultfield not in self.fields:
self.fields.append(defaultfield)
# keep a record of all field names found in the results
for fieldname in fields.keys():
if fieldname not in self.fields:
self.fields.append(fieldname)
# filter out unwanted fields
if self._fields != None:
fields = dict([(k, v) for k, v in fields.items()
if k in self._fields])
self.events.append(((evid.ts, evid.offset), defaultfield,
defaultvalue, fields))
logger.trace("added event %s to results" % evid)
finally:
for postingList in postingLists:
if postingList != None:
yield postingList.close()
for searcher in searchers:
yield searcher.close()
def next(self):
start = time.time()
searchers = []
postingLists = []
try:
# get a searcher and posting list for each index
for index in self._indices:
# we create a copy of the original query, which can possibly be optimized
# with index-specific knowledge.
query = copy.deepcopy(self._query)
# get the posting list to iterate through
searcher = yield index.newSearcher()
if not ISearcher.providedBy(searcher):
raise TypeError("searcher does not implement ISearcher")
query = yield query.optimizeMatcher(searcher)
logger.debug("optimized query for index '%s': %s" % (index.name,str(query)))
# if the query optimized out entirely, then skip to the next index
if query == None:
yield searcher.close()
continue
postingList = yield query.iterMatches(searcher, self._startId, self._endId)
if not IPostingList.providedBy(postingList):
raise TypeError("posting list does not implement IPostingList")
searchers.append(searcher)
postingLists.append(postingList)
if len(postingLists) == 0:
raise StopIteration()
# loop forever until we reach the search limit, we exhaust all of our
# posting lists, or we encounter an exception
currPostings = [(None,None,None) for i in range(len(postingLists))]
smallestList = 0
lastId = None
compar = (lambda x,y: cmp(y,x)) if self._reverse else cmp
while True:
# if we have reached our limit
if len(self.events) == self._limit:
self.runtime = time.time() - start
raise StopIteration()
# check each child iter for the lowest evid
for currList in range(len(postingLists)):
if currList == None:
smallestList = 0
# if the postingList is None, then its closed
if postingLists[currList] == None:
# FIXME: close the posting list and searcher
continue
# if current posting for this posting list was not consumed
if currPostings[currList] != (None,None,None):
continue
# otherwise get the next posting from the posting list
currPostings[currList] = yield postingLists[currList].nextPosting()
# if the next posting for this posting list is (None,None,None),
# then we are done with this posting list
if currPostings[currList] == (None,None,None):
postingList = postingLists[currList]
yield postingList.close()
postingLists[currList] = None
continue
# if the evid equals the last evid returned, then ignore it
if lastId != None and currPostings[currList][0] == lastId:
currPostings[currList] = (None,None,None)
continue
# we don't compare the first evid with itself
if currList == 0:
continue
# if the evid is the current smallest evid, then remember it
if (currPostings[smallestList] == (None,None,None) or
compar(currPostings[currList][0], currPostings[smallestList][0]) < 0):
smallestList = currList
# get the next posting
currList = None
evid,_,store = currPostings[smallestList]
# stop iterating if there are no more results
if evid == None:
self.runtime = time.time() - start
raise StopIteration()
# remember the last evid
lastId = evid
# forget the evid so we don't return it again
currPostings[smallestList] = (None,None,None)
# retrieve the event
if not IEventStore.providedBy(store):
raise TypeError("store does not implement IEventStore")
event = yield store.getEvent(evid)
defaultfield, defaultvalue, fields = event
if defaultfield not in self.fields:
self.fields.append(defaultfield)
# keep a record of all field names found in the results
for fieldname in fields.keys():
if fieldname not in self.fields:
self.fields.append(fieldname)
# filter out unwanted fields
if self._fields != None:
fields = dict([(k,v) for k,v in fields.items() if k in self._fields])
self.events.append(((evid.ts,evid.offset), defaultfield, defaultvalue, fields))
logger.trace("added event %s to results" % evid)
finally:
for postingList in postingLists:
if postingList != None:
yield postingList.close()
for searcher in searchers:
yield searcher.close()
def next(self):
try:
nlists = len(self._postingLists)
if nlists == 0:
raise StopIteration()
# check each child iter for the lowest evid
curr = 0
for i in range(nlists):
# if the postingList is None, then its closed
if self._postingLists[i] == None:
continue
# if current posting for this posting list was consumed, then
# get the next posting from the posting list
if self._currPostings[i] == (None, None, None):
self._currPostings[i] = self._postingLists[i].nextPosting()
# if the next posting for this posting list is (None,None,None),
# then we are done with this posting list
if self._currPostings[i] == (None, None, None):
self._postingLists[i] = None
continue
# if the evid equals the last evid returned, then ignore it
if self._lastId != None and self._currPostings[i][0] == self._lastId:
self._currPostings[i] = (None, None, None)
continue
# we don't compare the first evid with itself
if i == 0:
continue
# if the evid is the current smallest evid, then remember it
if (
self._currPostings[curr] == (None, None, None)
or self._cmp(self._currPostings[i][0], self._currPostings[curr][0]) < 0
):
curr = i
# stop iterating if there are no more results
if self._currPostings[curr] == (None, None, None):
raise StopIteration()
evid, tvalue, store = self._currPostings[curr]
# remember the last evid
self._lastId = evid
# forget the evid so we don't return it again
self._currPostings[curr] = (None, None, None)
# retrieve the event
if not IEventStore.providedBy(store):
raise TypeError("store does not implement IEventStore")
defaultfield, defaultvalue, fields = store.getEvent(evid)
if defaultfield not in self.fields:
self.fields.append(defaultfield)
# keep a record of all field names found in the results.
for fieldname in fields.keys():
if fieldname not in self.fields:
self.fields.append(fieldname)
# filter out unwanted fields
if self._fields != None:
fields = dict([(k, v) for k, v in fields.items() if k in self._fields])
self.events.append(((evid.ts, evid.offset), defaultfield, defaultvalue, fields))
logger.trace("added event %s to resultset" % evid)
# if we have reached our limit
self._count += 1
if self._count == self._limit:
raise StopIteration()
except:
self.close()
self.runtime = time.time() - self._start
logger.trace("retrieved %i events in %f seconds" % (len(self.events), self.runtime))
raise