def __init__(self, start, end, startexcl, endexcl):
"""
:param start: The start of the time range.
:type start: :class:`datetime.datetime`
:param end: The end of the time range.
:type end: :class:`datetime.datetime`
:param startexcl: If True, then the start of the range is exclusive.
:type startexcl: bool
:param endexcl: If True, then the end of the range is exclusive.
:type endexcl: bool
"""
if isinstance(start, datetime.datetime):
self.start = EVID.fromDatetime(start)
elif isinstance(start, EVID):
self.start = start
else:
raise TypeError(
"start must be datetime.datetime or terane.bier.evid.EVID")
if isinstance(end, datetime.datetime):
self.end = EVID.fromDatetime(end)
elif isinstance(end, EVID):
self.end = end
else:
raise TypeError(
"end must be datetime.datetime or terane.bier.evid.EVID")
self.startexcl = startexcl
self.endexcl = endexcl
def __init__(self, start, end, startexcl, endexcl):
"""
:param start: The start of the time range.
:type start: :class:`datetime.datetime`
:param end: The end of the time range.
:type end: :class:`datetime.datetime`
:param startexcl: If True, then the start of the range is exclusive.
:type startexcl: bool
:param endexcl: If True, then the end of the range is exclusive.
:type endexcl: bool
"""
if isinstance(start, datetime.datetime):
self.start = EVID.fromDatetime(start)
elif isinstance(start, EVID):
self.start = start
else:
raise TypeError("start must be datetime.datetime or terane.bier.evid.EVID")
if isinstance(end, datetime.datetime):
self.end = EVID.fromDatetime(end)
elif isinstance(end, EVID):
self.end = end
else:
raise TypeError("end must be datetime.datetime or terane.bier.evid.EVID")
self.startexcl = startexcl
self.endexcl = endexcl
def test_search_Every_reverse(self):
start = EVID.fromDatetime(*Bier_Searching_Tests.test_data[0][0:2])
end = EVID.fromDatetime(*Bier_Searching_Tests.test_data[-1][0:2])
period = Period(start, end, False, False)
resultset = yield searchIndices([self.output.getIndex(),], Every(), period, reverse=True).whenDone()
results = [EVID(*ev[0]) for ev in resultset.events]
test_data = sorted([EVID.fromDatetime(ts,offset) for ts,offset,_ in Bier_Searching_Tests.test_data], reverse=True)
self.assertEqual(results, test_data, "%s != %s" % (
[str(evid) for evid in results], [str(evid) for evid in test_data]))
def next(self):
writer = None
evid = EVID.fromEvent(self.event)
try:
# store the event
writer = yield self.index.newWriter()
if not IWriter.providedBy(writer):
raise TypeError("index writer does not implement IWriter")
fields = dict([(fn, v) for fn, ft, v in self.event])
logger.trace("[writer %s] creating event %s" % (self, evid))
yield writer.newEvent(evid, fields)
# process the value of each field in the event
for fieldname, fieldtype, value in self.event:
logger.trace("[writer %s] using field %s:%s" %
(self, fieldname, fieldtype))
field = yield writer.getField(fieldname, fieldtype)
# store a posting for each term in each field
for term, meta in field.parseValue(value):
logger.trace("[writer %s] creating posting %s:%s:%s" %
(self, field, term, evid))
yield writer.newPosting(field, term, evid, meta)
logger.debug("[writer %s] committed event %s" % (self, str(evid)))
finally:
if writer != None:
yield writer.close()
def test_search_Every(self):
start = EVID.fromDatetime(*Bier_Searching_Tests.test_data[0][0:2])
end = EVID.fromDatetime(*Bier_Searching_Tests.test_data[-1][0:2])
period = Period(start, end, False, False)
resultset = yield searchIndices([
self.output.getIndex(),
], Every(), period).whenDone()
results = [EVID(*ev[0]) for ev in resultset.events]
test_data = [
EVID.fromDatetime(ts, offset)
for ts, offset, _ in Bier_Searching_Tests.test_data
]
self.assertEqual(
results, test_data, "%s != %s" %
([str(evid)
for evid in results], [str(evid) for evid in test_data]))
def _nextPosting(postingList):
posting = (None, None, None)
if postingList._postings == None:
return posting
try:
key, value = postingList._postings.next()
# posting key consists of: fieldname, fieldtype, term, ts, id
if len(key) == 5:
evid = EVID(key[3], key[4])
# term key consists of: ts, id
else:
evid = EVID(key[0], key[1])
posting = (evid, value, postingList._searcher)
except StopIteration:
postingList._close()
return posting
def _skipPosting(postingList, targetId):
prefix = [
postingList._field.fieldname, postingList._field.fieldtype
]
nextPosting = (None, None, None)
# find the next posting closest to the smallestId
for termKey, termValue in postingList._terms:
# check whether we should exclude this term
if postingList._startEx and postingList._startEx == termKey[2]:
continue
if postingList._endEx and postingList._endEx == termKey[2]:
continue
# check if we have iterated past the last term for the specified field
if termKey[0:2] != prefix:
break
# jump to the next closest key
targetKey = termKey + [targetId.ts, targetId.offset]
try:
postingKey, postingValue = postingList._postings.skip(
targetKey, False)
except IndexError:
continue
logger.trace("skip range posting: %s" % postingKey)
currId = EVID(postingKey[3], postingKey[4])
nextPosting = (currId, postingValue, postingList._searcher)
# rewind the iterators
postingList._terms.reset()
postingList._postings.reset()
return nextPosting
def _nextPosting(postingList):
prefix = [
postingList._field.fieldname, postingList._field.fieldtype
]
smallestId = None
nextPosting = (None, None, None)
# find the next posting closest to the smallestId
for termKey, termValue in postingList._terms:
# check whether we should exclude this term
if postingList._startEx and postingList._startEx == termKey[2]:
continue
if postingList._endEx and postingList._endEx == termKey[2]:
continue
# check if we have iterated past the last term for the specified field
if termKey[0:2] != prefix:
break
# jump to the next closest key
if postingList._lastId == None:
closestKey = termKey + [
postingList._startId.ts, postingList._startId.offset
]
else:
if postingList._reverse:
closestKey = termKey + [
postingList._lastId.ts,
postingList._lastId.offset - 1
]
else:
closestKey = termKey + [
postingList._lastId.ts,
postingList._lastId.offset + 1
]
try:
postingKey, postingValue = postingList._postings.skip(
closestKey, True)
except IndexError:
continue
logger.trace("next range posting: %s" % postingKey)
currId = EVID(postingKey[3], postingKey[4])
# check whether this posting is the smallest so far
if ((smallestId == None
or postingList._cmp(currId, smallestId) < 0)
and termKey == postingKey[0:3]
and postingList._cmp(currId, postingList._endId) <= 0):
smallestId = currId
nextPosting = (smallestId, postingValue,
postingList._searcher)
# no more ids, we are done
if smallestId == None:
return (None, None, None)
# rewind the iterators
postingList._terms.reset()
postingList._postings.reset()
postingList._lastId = smallestId
return nextPosting
def _getResult(self, results):
"""
Append each search result into the ResultsListbox.
"""
try:
self._meta = results.pop(0)
for evid,event in results:
self._results.append(EVID.fromString(evid), event)
console.redraw()
except Exception, e:
logger.exception(e)
def _getResult(self, result):
"""
Append each search result into the ResultsListbox.
"""
try:
self._meta = result['meta']
data = result['data']
for evid, defaultfield, defaultvalue, fields in data:
evid = EVID(evid[0], evid[1])
self._results.append(evid, defaultfield, defaultvalue, fields)
console.redraw()
except Exception, e:
logger.exception(e)
def _skipPosting(postingList, targetId):
posting = (None, None, None)
if postingList._postings == None:
return posting
try:
target = [
postingList._field.fieldname, postingList._field.fieldtype,
postingList._term, targetId.ts, targetId.offset
]
key, value = postingList._postings.skip(target)
# posting key consists of: fieldname, fieldtype, term, ts, id
if len(key) == 5:
evid = EVID(key[3], key[4])
# term key consists of: ts, id
else:
evid = EVID(key[0], key[1])
posting = (evid, value, postingList._searcher)
except IndexError:
pass
except StopIteration:
postingList._close()
return posting
def test_read_write_index(self):
contract = Contract().sign()
for ts,offset,message in Output_Store_Tests.test_data:
event = Event(ts, offset)
event[contract.field_message] = message
self.output.receiveEvent(event)
# read back from the index
index = self.output.getIndex()
startId = EVID.fromDatetime(*Output_Store_Tests.test_data[0][0:2])
endId = EVID.fromDatetime(*Output_Store_Tests.test_data[-1][0:2])
searcher = index.newSearcher()
try:
npostings = searcher.postingsLength(None, None, startId, endId)
self.assertTrue(npostings == len(Output_Store_Tests.test_data))
i = searcher.iterPostings(None, None, startId, endId)
postings = []
while True:
posting = i.nextPosting()
if posting == (None, None, None):
break
postings.append(posting)
finally:
searcher.close()
def _getResult(self, results):
"""
Append each search result into the ResultsListbox.
"""
try:
meta = results.pop(0)
self._lastId = meta['lastId']
logger.debug("tail returned %i results, last id is %s" % (len(results), self._lastId))
if len(results) > 0:
for evid,event in results:
self._results.append(EVID.fromString(evid), event)
console.redraw()
from twisted.internet import reactor
self._delayed = reactor.callLater(self.interval, self._tail)
except Exception, e:
logger.exception(e)
def writeEventToIndex(event, index):
"""
:param event:
:type event:
:param index:
:type index:
:returns: The EVID of the new event.
:rtype: :class:`terane.bier.evid.EVID`
"""
# verify that the index provides the appropriate interface
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
# get the current schema
schema = index.getSchema()
if not ISchema.providedBy(schema):
raise TypeError("index schema does not implement ISchema")
# create a new transactional writer
writer = index.newWriter()
if not IWriter.providedBy(writer):
raise TypeError("index writer does not implement IWriter")
writer.begin()
# update the index in the context of a writer
try:
# create a new event identifier
evid = EVID.fromEvent(event)
# process the value of each field in the event
fields = {}
for fieldname, fieldtype, value in event:
if not schema.hasField(fieldname, fieldtype):
schema.addField(fieldname, fieldtype)
field = schema.getField(fieldname, fieldtype)
# update the field with the event value
for term, meta in field.parseValue(value):
writer.newPosting(field, term, evid, meta)
fields[fieldname] = value
# store the document data
writer.newEvent(evid, fields)
# if an exception was raised, then abort the transaction
except:
writer.abort()
raise
# otherwise commit the transaction and return the event id
writer.commit()
return evid
def printResult(self, results):
logger.debug("XMLRPC result: %s" % str(results))
meta = results["meta"]
data = results["data"]
if len(data) > 0:
for evid, event in data:
evid = EVID.fromString(evid)
ts = datetime.datetime.fromtimestamp(evid.ts, dateutil.tz.tzutc())
if self.tz:
ts = ts.astimezone(self.tz)
print "%s: %s" % (ts.strftime("%d %b %Y %H:%M:%S %Z"), event["default"])
if self.longfmt:
del event["default"]
for field, value in sorted(event.items(), key=lambda x: x[0]):
if self.fields and field not in self.fields:
continue
print "\t%s=%s" % (field, value)
return meta["lastId"]
def _getResult(self, results):
"""
Append each search result into the ResultsListbox.
"""
try:
meta = results.pop(0)
self._lastId = meta['lastId']
logger.debug("tail returned %i results, last id is %s" %
(len(results), self._lastId))
if len(results) > 0:
for evid, defaultfield, defaultvalue, fields in results:
evid = EVID(evid[0], evid[1])
self._results.append(evid, defaultfield, defaultvalue,
fields)
console.redraw()
from twisted.internet import reactor
self._delayed = reactor.callLater(self.interval, self._tail)
except Exception, e:
logger.exception(e)
def printResult(self, results):
logger.debug("XMLRPC result: %s" % str(results))
meta = results['meta']
data = results['data']
if len(data) > 0:
for evid, event in data:
evid = EVID.fromString(evid)
ts = datetime.datetime.fromtimestamp(evid.ts,
dateutil.tz.tzutc())
if self.tz:
ts = ts.astimezone(self.tz)
print "%s: %s" % (ts.strftime("%d %b %Y %H:%M:%S %Z"),
event['default'])
if self.longfmt:
del event['default']
for field, value in sorted(event.items(),
key=lambda x: x[0]):
if self.fields and field not in self.fields:
continue
print "\t%s=%s" % (field, value)
return meta['lastId']
def next(self):
writer = None
evid = EVID.fromEvent(self.event)
try:
# store the event
writer = yield self.index.newWriter()
if not IWriter.providedBy(writer):
raise TypeError("index writer does not implement IWriter")
fields = dict([(fn,v) for fn,ft,v in self.event])
logger.trace("[writer %s] creating event %s" % (self,evid))
yield writer.newEvent(evid, fields)
# process the value of each field in the event
for fieldname, fieldtype, value in self.event:
logger.trace("[writer %s] using field %s:%s" % (self,fieldname,fieldtype))
field = yield writer.getField(fieldname, fieldtype)
# store a posting for each term in each field
for term,meta in field.parseValue(value):
logger.trace("[writer %s] creating posting %s:%s:%s" % (self,field,term,evid))
yield writer.newPosting(field, term, evid, meta)
logger.debug("[writer %s] committed event %s" % (self,str(evid)))
finally:
if writer != None:
yield writer.close()
:param fields: A list of fields to return in the results, or None to return all fields.
:type fields: list
:returns: A Deferred object which receives the results.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
# look up the named indices
if indices == None:
indices = tuple(self._indexstore.iterSearchableIndices())
else:
try:
indices = tuple(self._indexstore.getSearchableIndex(name) for name in indices)
except KeyError, e:
raise QueryExecutionError("unknown index '%s'" % e)
# if lastId is 0, return the id of the latest document
if lastId == None:
return QueryResult({'runtime': 0.0, 'lastId': str(EVID.fromDatetime())}, [])
try:
lastId = EVID.fromString(lastId)
except:
raise QueryExecutionError("invalid lastId '%s'" % str(lastId))
# check that limit is > 0
if limit < 1:
raise QueryExecutionError("limit must be greater than 0")
query = parseTailQuery(query)
logger.trace("tail query: %s" % query)
period = Period(lastId, EVID.fromString(EVID.MAX_ID), True, False)
logger.trace("tail period: %s" % period)
# query each index, and return the results
task = searchIndices(indices, query, period, None, False, fields, limit)
def _returnTailResult(result, lastId=None):
if isinstance(result, Failure) and result.check(SearcherError):
# look up the named indices
if indices == None:
indices = tuple(self._indexstore.iterSearchableIndices())
else:
try:
indices = tuple(
self._indexstore.getSearchableIndex(name)
for name in indices)
except KeyError, e:
raise QueryExecutionError("unknown index '%s'" % e)
# if lastId is 0, return the id of the latest document
if lastId == None:
return QueryResult(
{
'runtime': 0.0,
'lastId': str(EVID.fromDatetime())
}, [])
try:
lastId = EVID.fromString(lastId)
except:
raise QueryExecutionError("invalid lastId '%s'" % str(lastId))
# check that limit is > 0
if limit < 1:
raise QueryExecutionError("limit must be greater than 0")
query = parseTailQuery(query)
logger.trace("tail query: %s" % query)
period = Period(lastId, EVID.fromString(EVID.MAX_ID), True, False)
logger.trace("tail period: %s" % period)
# query each index, and return the results
def _returnTailResult(result, lastId=None):