def searchIndices(indices, query, period, lastId=None, reverse=False, fields=None, limit=100): """ Search the specified indices for events matching the specified query. :param indices: A list of indices to search. :type indices: A list of objects implementing :class:`terane.bier.index.IIndex` :param query: The programmatic query to use for searching the indices. :type query: An object implementing :class:`terane.bier.searching.IQuery` :param period: The period within which the search is constrained. :type period: :class:`terane.bier.searching.Period` :param lastId: The real key to start iterating from. :type lastId: :class:`terane.bier.evid.EVID` :param reverse: If True, then reverse the order of events. :type reverse: bool :param fields: If not None, then only return the specified fields of each event. :type fields: list or None :param limit: Only returned the specified number of events. :type limit: int :returns: A CooperativeTask which contains a Deferred and manages the search task. :rtype: :class:`twisted.internet.task.CooperativeTask` """ start = time.time() # determine the evids to use as start and end keys if reverse == False: startId, endId = period.getRange() else: endId, startId = period.getRange() if lastId != None: if not lastId in period: raise SearcherError("lastId %s is not within period" % lastId) startId = lastId # search each index separately, then merge the results try: searchers = [] postingLists = [] for index in indices: if not IIndex.providedBy(index): raise TypeError("index does not implement IIndex") # we create a copy of the original query, which can possibly be optimized # with index-specific knowledge. _query = copy.deepcopy(query) try: _query = _query.optimizeMatcher(index) except NotImplementedError, e: raise SearcherError(str(e)) logger.debug("optimized query for index '%s': %s" % (index.name, str(_query))) # if the query optimized out entirely, then skip to the next index if _query == None: continue # get the posting list to iterate through searcher = index.newSearcher() if not ISearcher.providedBy(searcher): raise TypeError("searcher does not implement ISearcher") postingList = _query.iterMatches(searcher, startId, endId) if not IPostingList.providedBy(postingList): raise TypeError("posting list does not implement IPostingList") searchers.append(searcher) postingLists.append(postingList) # return a cooperative task return cooperate(ResultSet(searchers, postingLists, start, reverse, fields, limit))
def __init__(self, event, index): """ :param event: The event to write. :type event: :class:`terane.bier.event.Event` :param index: The index which will receive the event. :type index: Object implementing :class:`terane.bier.interfaces.IIndex` """ if not isinstance(event, Event): raise TypeError("event is not an Event") self.event = event # verify that the index provides the appropriate interface if not IIndex.providedBy(index): raise TypeError("index does not implement IIndex") self.index = index
def writeEventToIndex(event, index): """ :param event: :type event: :param index: :type index: :returns: The EVID of the new event. :rtype: :class:`terane.bier.evid.EVID` """ # verify that the index provides the appropriate interface if not IIndex.providedBy(index): raise TypeError("index does not implement IIndex") # get the current schema schema = index.getSchema() if not ISchema.providedBy(schema): raise TypeError("index schema does not implement ISchema") # create a new transactional writer writer = index.newWriter() if not IWriter.providedBy(writer): raise TypeError("index writer does not implement IWriter") writer.begin() # update the index in the context of a writer try: # create a new event identifier evid = EVID.fromEvent(event) # process the value of each field in the event fields = {} for fieldname, fieldtype, value in event: if not schema.hasField(fieldname, fieldtype): schema.addField(fieldname, fieldtype) field = schema.getField(fieldname, fieldtype) # update the field with the event value for term, meta in field.parseValue(value): writer.newPosting(field, term, evid, meta) fields[fieldname] = value # store the document data writer.newEvent(evid, fields) # if an exception was raised, then abort the transaction except: writer.abort() raise # otherwise commit the transaction and return the event id writer.commit() return evid
def __init__(self, indices, query, period, lastId=None, reverse=False, fields=None, limit=100): """ :param indices: A list of indices to search. :type indices: A list of objects implementing :class:`terane.bier.index.IIndex` :param query: The programmatic query to use for searching the indices. :type query: An object implementing :class:`terane.bier.searching.IQuery` :param period: The period within which the search is constrained. :type period: :class:`terane.bier.searching.Period` :param lastId: The real key to start iterating from. :type lastId: :class:`terane.bier.evid.EVID` :param reverse: If True, then reverse the order of events. :type reverse: bool :param fields: If not None, then only return the specified fields of each event. :type fields: list or None :param limit: Only returned the specified number of events. :type limit: int """ # determine the evids to use as start and end keys if reverse == False: self._startId, self._endId = period.getRange() else: self._endId, self._startId = period.getRange() if lastId != None: if not lastId in period: raise SearcherError("lastId %s is not within period" % lastId) self._startId = lastId self._lastId = lastId for index in indices: if not IIndex.providedBy(index): raise TypeError( "one or more indices does not implement IIndex") self._indices = indices self._query = query self._reverse = reverse self._fields = fields self._limit = limit self.events = [] self.fields = [] self.runtime = 0.0
def __init__(self, indices, query, period, lastId=None, reverse=False, fields=None, limit=100): """ :param indices: A list of indices to search. :type indices: A list of objects implementing :class:`terane.bier.index.IIndex` :param query: The programmatic query to use for searching the indices. :type query: An object implementing :class:`terane.bier.searching.IQuery` :param period: The period within which the search is constrained. :type period: :class:`terane.bier.searching.Period` :param lastId: The real key to start iterating from. :type lastId: :class:`terane.bier.evid.EVID` :param reverse: If True, then reverse the order of events. :type reverse: bool :param fields: If not None, then only return the specified fields of each event. :type fields: list or None :param limit: Only returned the specified number of events. :type limit: int """ # determine the evids to use as start and end keys if reverse == False: self._startId, self._endId = period.getRange() else: self._endId, self._startId = period.getRange() if lastId != None: if not lastId in period: raise SearcherError("lastId %s is not within period" % lastId) self._startId = lastId self._lastId = lastId for index in indices: if not IIndex.providedBy(index): raise TypeError("one or more indices does not implement IIndex") self._indices = indices self._query = query self._reverse = reverse self._fields = fields self._limit = limit self.events = [] self.fields = [] self.runtime = 0.0