def searchIndices(indices, query, period, lastId=None, reverse=False, fields=None, limit=100):
"""
Search the specified indices for events matching the specified query.
:param indices: A list of indices to search.
:type indices: A list of objects implementing :class:`terane.bier.index.IIndex`
:param query: The programmatic query to use for searching the indices.
:type query: An object implementing :class:`terane.bier.searching.IQuery`
:param period: The period within which the search is constrained.
:type period: :class:`terane.bier.searching.Period`
:param lastId: The real key to start iterating from.
:type lastId: :class:`terane.bier.evid.EVID`
:param reverse: If True, then reverse the order of events.
:type reverse: bool
:param fields: If not None, then only return the specified fields of each event.
:type fields: list or None
:param limit: Only returned the specified number of events.
:type limit: int
:returns: A CooperativeTask which contains a Deferred and manages the search task.
:rtype: :class:`twisted.internet.task.CooperativeTask`
"""
start = time.time()
# determine the evids to use as start and end keys
if reverse == False:
startId, endId = period.getRange()
else:
endId, startId = period.getRange()
if lastId != None:
if not lastId in period:
raise SearcherError("lastId %s is not within period" % lastId)
startId = lastId
# search each index separately, then merge the results
try:
searchers = []
postingLists = []
for index in indices:
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
# we create a copy of the original query, which can possibly be optimized
# with index-specific knowledge.
_query = copy.deepcopy(query)
try:
_query = _query.optimizeMatcher(index)
except NotImplementedError, e:
raise SearcherError(str(e))
logger.debug("optimized query for index '%s': %s" % (index.name, str(_query)))
# if the query optimized out entirely, then skip to the next index
if _query == None:
continue
# get the posting list to iterate through
searcher = index.newSearcher()
if not ISearcher.providedBy(searcher):
raise TypeError("searcher does not implement ISearcher")
postingList = _query.iterMatches(searcher, startId, endId)
if not IPostingList.providedBy(postingList):
raise TypeError("posting list does not implement IPostingList")
searchers.append(searcher)
postingLists.append(postingList)
# return a cooperative task
return cooperate(ResultSet(searchers, postingLists, start, reverse, fields, limit))
def __init__(self, event, index):
"""
:param event: The event to write.
:type event: :class:`terane.bier.event.Event`
:param index: The index which will receive the event.
:type index: Object implementing :class:`terane.bier.interfaces.IIndex`
"""
if not isinstance(event, Event):
raise TypeError("event is not an Event")
self.event = event
# verify that the index provides the appropriate interface
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
self.index = index
def __init__(self, event, index):
"""
:param event: The event to write.
:type event: :class:`terane.bier.event.Event`
:param index: The index which will receive the event.
:type index: Object implementing :class:`terane.bier.interfaces.IIndex`
"""
if not isinstance(event, Event):
raise TypeError("event is not an Event")
self.event = event
# verify that the index provides the appropriate interface
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
self.index = index
def writeEventToIndex(event, index):
"""
:param event:
:type event:
:param index:
:type index:
:returns: The EVID of the new event.
:rtype: :class:`terane.bier.evid.EVID`
"""
# verify that the index provides the appropriate interface
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
# get the current schema
schema = index.getSchema()
if not ISchema.providedBy(schema):
raise TypeError("index schema does not implement ISchema")
# create a new transactional writer
writer = index.newWriter()
if not IWriter.providedBy(writer):
raise TypeError("index writer does not implement IWriter")
writer.begin()
# update the index in the context of a writer
try:
# create a new event identifier
evid = EVID.fromEvent(event)
# process the value of each field in the event
fields = {}
for fieldname, fieldtype, value in event:
if not schema.hasField(fieldname, fieldtype):
schema.addField(fieldname, fieldtype)
field = schema.getField(fieldname, fieldtype)
# update the field with the event value
for term, meta in field.parseValue(value):
writer.newPosting(field, term, evid, meta)
fields[fieldname] = value
# store the document data
writer.newEvent(evid, fields)
# if an exception was raised, then abort the transaction
except:
writer.abort()
raise
# otherwise commit the transaction and return the event id
writer.commit()
return evid
def __init__(self,
indices,
query,
period,
lastId=None,
reverse=False,
fields=None,
limit=100):
"""
:param indices: A list of indices to search.
:type indices: A list of objects implementing :class:`terane.bier.index.IIndex`
:param query: The programmatic query to use for searching the indices.
:type query: An object implementing :class:`terane.bier.searching.IQuery`
:param period: The period within which the search is constrained.
:type period: :class:`terane.bier.searching.Period`
:param lastId: The real key to start iterating from.
:type lastId: :class:`terane.bier.evid.EVID`
:param reverse: If True, then reverse the order of events.
:type reverse: bool
:param fields: If not None, then only return the specified fields of each event.
:type fields: list or None
:param limit: Only returned the specified number of events.
:type limit: int
"""
# determine the evids to use as start and end keys
if reverse == False:
self._startId, self._endId = period.getRange()
else:
self._endId, self._startId = period.getRange()
if lastId != None:
if not lastId in period:
raise SearcherError("lastId %s is not within period" % lastId)
self._startId = lastId
self._lastId = lastId
for index in indices:
if not IIndex.providedBy(index):
raise TypeError(
"one or more indices does not implement IIndex")
self._indices = indices
self._query = query
self._reverse = reverse
self._fields = fields
self._limit = limit
self.events = []
self.fields = []
self.runtime = 0.0
def __init__(self, indices, query, period, lastId=None, reverse=False, fields=None, limit=100):
"""
:param indices: A list of indices to search.
:type indices: A list of objects implementing :class:`terane.bier.index.IIndex`
:param query: The programmatic query to use for searching the indices.
:type query: An object implementing :class:`terane.bier.searching.IQuery`
:param period: The period within which the search is constrained.
:type period: :class:`terane.bier.searching.Period`
:param lastId: The real key to start iterating from.
:type lastId: :class:`terane.bier.evid.EVID`
:param reverse: If True, then reverse the order of events.
:type reverse: bool
:param fields: If not None, then only return the specified fields of each event.
:type fields: list or None
:param limit: Only returned the specified number of events.
:type limit: int
"""
# determine the evids to use as start and end keys
if reverse == False:
self._startId, self._endId = period.getRange()
else:
self._endId, self._startId = period.getRange()
if lastId != None:
if not lastId in period:
raise SearcherError("lastId %s is not within period" % lastId)
self._startId = lastId
self._lastId = lastId
for index in indices:
if not IIndex.providedBy(index):
raise TypeError("one or more indices does not implement IIndex")
self._indices = indices
self._query = query
self._reverse = reverse
self._fields = fields
self._limit = limit
self.events = []
self.fields = []
self.runtime = 0.0
