Пример #1
0
 def next(self):
     start = time.time()
     searchers = []
     postingLists = []
     try:
         # get a searcher and posting list for each index
         for index in self._indices:
             # we create a copy of the original query, which can possibly be optimized
             # with index-specific knowledge.
             query = copy.deepcopy(self._query)
             # get the posting list to iterate through
             searcher = yield index.newSearcher()
             if not ISearcher.providedBy(searcher):
                 raise TypeError("searcher does not implement ISearcher")
             query = yield query.optimizeMatcher(searcher)
             logger.debug("optimized query for index '%s': %s" %
                          (index.name, str(query)))
             # if the query optimized out entirely, then skip to the next index
             if query == None:
                 yield searcher.close()
                 continue
             postingList = yield query.iterMatches(searcher, self._startId,
                                                   self._endId)
             if not IPostingList.providedBy(postingList):
                 raise TypeError(
                     "posting list does not implement IPostingList")
             searchers.append(searcher)
             postingLists.append(postingList)
         if len(postingLists) == 0:
             raise StopIteration()
         # loop forever until we reach the search limit, we exhaust all of our
         # posting lists, or we encounter an exception
         currPostings = [(None, None, None)
                         for i in range(len(postingLists))]
         smallestList = 0
         lastId = None
         compar = (lambda x, y: cmp(y, x)) if self._reverse else cmp
         while True:
             # if we have reached our limit
             if len(self.events) == self._limit:
                 self.runtime = time.time() - start
                 raise StopIteration()
             # check each child iter for the lowest evid
             for currList in range(len(postingLists)):
                 if currList == None:
                     smallestList = 0
                 # if the postingList is None, then its closed
                 if postingLists[currList] == None:
                     # FIXME: close the posting list and searcher
                     continue
                 # if current posting for this posting list was not consumed
                 if currPostings[currList] != (None, None, None):
                     continue
                 # otherwise get the next posting from the posting list
                 currPostings[currList] = yield postingLists[
                     currList].nextPosting()
                 # if the next posting for this posting list is (None,None,None),
                 # then we are done with this posting list
                 if currPostings[currList] == (None, None, None):
                     postingList = postingLists[currList]
                     yield postingList.close()
                     postingLists[currList] = None
                     continue
                 # if the evid equals the last evid returned, then ignore it
                 if lastId != None and currPostings[currList][0] == lastId:
                     currPostings[currList] = (None, None, None)
                     continue
                 # we don't compare the first evid with itself
                 if currList == 0:
                     continue
                 # if the evid is the current smallest evid, then remember it
                 if (currPostings[smallestList] == (None, None, None)
                         or compar(currPostings[currList][0],
                                   currPostings[smallestList][0]) < 0):
                     smallestList = currList
             # get the next posting
             currList = None
             evid, _, store = currPostings[smallestList]
             # stop iterating if there are no more results
             if evid == None:
                 self.runtime = time.time() - start
                 raise StopIteration()
             # remember the last evid
             lastId = evid
             # forget the evid so we don't return it again
             currPostings[smallestList] = (None, None, None)
             # retrieve the event
             if not IEventStore.providedBy(store):
                 raise TypeError("store does not implement IEventStore")
             event = yield store.getEvent(evid)
             defaultfield, defaultvalue, fields = event
             if defaultfield not in self.fields:
                 self.fields.append(defaultfield)
             # keep a record of all field names found in the results
             for fieldname in fields.keys():
                 if fieldname not in self.fields:
                     self.fields.append(fieldname)
             # filter out unwanted fields
             if self._fields != None:
                 fields = dict([(k, v) for k, v in fields.items()
                                if k in self._fields])
             self.events.append(((evid.ts, evid.offset), defaultfield,
                                 defaultvalue, fields))
             logger.trace("added event %s to results" % evid)
     finally:
         for postingList in postingLists:
             if postingList != None:
                 yield postingList.close()
         for searcher in searchers:
             yield searcher.close()
Пример #2
0
 def next(self):
     start = time.time()
     searchers = []
     postingLists = []
     try:
         # get a searcher and posting list for each index
         for index in self._indices:
             # we create a copy of the original query, which can possibly be optimized
             # with index-specific knowledge.
             query = copy.deepcopy(self._query)
             # get the posting list to iterate through
             searcher = yield index.newSearcher()
             if not ISearcher.providedBy(searcher):
                 raise TypeError("searcher does not implement ISearcher")
             query = yield query.optimizeMatcher(searcher)
             logger.debug("optimized query for index '%s': %s" % (index.name,str(query)))
             # if the query optimized out entirely, then skip to the next index
             if query == None:
                 yield searcher.close()
                 continue
             postingList = yield query.iterMatches(searcher, self._startId, self._endId)
             if not IPostingList.providedBy(postingList):
                 raise TypeError("posting list does not implement IPostingList")
             searchers.append(searcher)
             postingLists.append(postingList)
         if len(postingLists) == 0:
             raise StopIteration()
         # loop forever until we reach the search limit, we exhaust all of our
         # posting lists, or we encounter an exception
         currPostings = [(None,None,None) for i in range(len(postingLists))]
         smallestList = 0
         lastId = None
         compar = (lambda x,y: cmp(y,x)) if self._reverse else cmp 
         while True:
             # if we have reached our limit
             if len(self.events) == self._limit:
                 self.runtime = time.time() - start
                 raise StopIteration()
             # check each child iter for the lowest evid
             for currList in range(len(postingLists)):
                 if currList == None:
                     smallestList = 0
                 # if the postingList is None, then its closed
                 if postingLists[currList] == None:
                     # FIXME: close the posting list and searcher
                     continue
                 # if current posting for this posting list was not consumed
                 if currPostings[currList] != (None,None,None):
                     continue
                 # otherwise get the next posting from the posting list
                 currPostings[currList] = yield postingLists[currList].nextPosting()
                 # if the next posting for this posting list is (None,None,None),
                 # then we are done with this posting list
                 if currPostings[currList] == (None,None,None):
                     postingList = postingLists[currList]
                     yield postingList.close()
                     postingLists[currList] = None
                     continue
                 # if the evid equals the last evid returned, then ignore it
                 if lastId != None and currPostings[currList][0] == lastId:
                     currPostings[currList] = (None,None,None)
                     continue
                 # we don't compare the first evid with itself
                 if currList == 0:
                     continue
                 # if the evid is the current smallest evid, then remember it
                 if (currPostings[smallestList] == (None,None,None) or
                   compar(currPostings[currList][0], currPostings[smallestList][0]) < 0):
                     smallestList = currList
             # get the next posting
             currList = None
             evid,_,store = currPostings[smallestList]
             # stop iterating if there are no more results
             if evid == None:
                 self.runtime = time.time() - start
                 raise StopIteration()
             # remember the last evid
             lastId = evid
             # forget the evid so we don't return it again
             currPostings[smallestList] = (None,None,None)
             # retrieve the event
             if not IEventStore.providedBy(store):
                 raise TypeError("store does not implement IEventStore")
             event = yield store.getEvent(evid)
             defaultfield, defaultvalue, fields = event
             if defaultfield not in self.fields:
                 self.fields.append(defaultfield)
             # keep a record of all field names found in the results
             for fieldname in fields.keys():
                 if fieldname not in self.fields:
                     self.fields.append(fieldname)
             # filter out unwanted fields
             if self._fields != None:
                 fields = dict([(k,v) for k,v in fields.items() if k in self._fields])
             self.events.append(((evid.ts,evid.offset), defaultfield, defaultvalue, fields))
             logger.trace("added event %s to results" % evid)
     finally:
         for postingList in postingLists:
             if postingList != None:
                 yield postingList.close()
         for searcher in searchers:
             yield searcher.close()
Пример #3
0
 def next(self):
     try:
         nlists = len(self._postingLists)
         if nlists == 0:
             raise StopIteration()
         # check each child iter for the lowest evid
         curr = 0
         for i in range(nlists):
             # if the postingList is None, then its closed
             if self._postingLists[i] == None:
                 continue
             # if current posting for this posting list was consumed, then
             # get the next posting from the posting list
             if self._currPostings[i] == (None, None, None):
                 self._currPostings[i] = self._postingLists[i].nextPosting()
             # if the next posting for this posting list is (None,None,None),
             # then we are done with this posting list
             if self._currPostings[i] == (None, None, None):
                 self._postingLists[i] = None
                 continue
             # if the evid equals the last evid returned, then ignore it
             if self._lastId != None and self._currPostings[i][0] == self._lastId:
                 self._currPostings[i] = (None, None, None)
                 continue
             # we don't compare the first evid with itself
             if i == 0:
                 continue
             # if the evid is the current smallest evid, then remember it
             if (
                 self._currPostings[curr] == (None, None, None)
                 or self._cmp(self._currPostings[i][0], self._currPostings[curr][0]) < 0
             ):
                 curr = i
         # stop iterating if there are no more results
         if self._currPostings[curr] == (None, None, None):
             raise StopIteration()
         evid, tvalue, store = self._currPostings[curr]
         # remember the last evid
         self._lastId = evid
         # forget the evid so we don't return it again
         self._currPostings[curr] = (None, None, None)
         # retrieve the event
         if not IEventStore.providedBy(store):
             raise TypeError("store does not implement IEventStore")
         defaultfield, defaultvalue, fields = store.getEvent(evid)
         if defaultfield not in self.fields:
             self.fields.append(defaultfield)
         # keep a record of all field names found in the results.
         for fieldname in fields.keys():
             if fieldname not in self.fields:
                 self.fields.append(fieldname)
         # filter out unwanted fields
         if self._fields != None:
             fields = dict([(k, v) for k, v in fields.items() if k in self._fields])
         self.events.append(((evid.ts, evid.offset), defaultfield, defaultvalue, fields))
         logger.trace("added event %s to resultset" % evid)
         # if we have reached our limit
         self._count += 1
         if self._count == self._limit:
             raise StopIteration()
     except:
         self.close()
         self.runtime = time.time() - self._start
         logger.trace("retrieved %i events in %f seconds" % (len(self.events), self.runtime))
         raise