def rotateSegments(self, segRotation, segRetention):
"""
Allocate a new Segment, making it the new current segment.
"""
# if the current segment contains more events than specified by
# segRotation, then rotate the index to generate a new segment.
if segRotation > 0 and self._currentSize >= segRotation:
with self.new_txn() as txn:
segmentId = self.new_segment(txn)
segment = Segment(txn, self, segmentId)
segment.set_meta(txn, u'created-on', int(time.time()))
last_update = {
u'size': 0,
u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
u'last-modified': 0
}
segment.set_meta(txn, u'last-update', last_update)
self._segments.append(segment)
self._current = segment
self._currentSize = 0
logger.debug("rotated current segment, new segment is %s" % segment.fullName)
# if the index contains more segments than specified by segRetention,
# then delete the oldest segment.
if segRetention > 0:
if len(self._segments) > segRetention:
for segment in self._segments[0:len(self._segments)-segRetention]:
self.delete(segment)
def __init__(self, output):
self.name = output._indexName
self._env = output._plugin._env
backend.Index.__init__(self, self._env, self.name)
self._segmentLock = Lock()
self._segments = []
self._current = None
self._fieldLock = Lock()
self._fieldstore = output._fieldstore
self._fields = {}
self._indexUUID = None
try:
# load index metadata
with self.new_txn() as txn:
try:
self._indexUUID = UUID(self.get_meta(txn, u'uuid', RMW=True))
except KeyError:
self._indexUUID = uuid4()
self.set_meta(txn, u'uuid', unicode(self._indexUUID))
# load schema
with self.new_txn() as txn:
for fieldname,fieldspec in self.iter_fields(txn):
self._fields[fieldname] = pickle.loads(str(fieldspec))
# verify that the field type is consistent
for fieldtype,stored in self._fields[fieldname].items():
field = self._fieldstore.getField(fieldtype)
if not stored.field.__class__ == field.__class__:
raise SchemaError("schema field %s:%s does not match registered type %s" % (
fieldname, fieldtype, field.__class__.__name__))
# load data segments
indexSize = 0
with self.new_txn() as txn:
for segmentName,segmentUUID in self.iter_segments(txn):
segment = Segment(self._env, txn, segmentName)
try:
foundUUID = segment.get_meta(txn, u'uuid')
except KeyError:
foundUUID = None
if foundUUID == None or segmentUUID != foundUUID:
logger.debug("index segment %s has invalid UUID %s" % (segmentName, foundUUID))
segment.close()
else:
self._segments.append(segment)
logger.debug("opened index segment '%s'" % segmentName)
# if the index has no segments, create one
if self._segments == []:
self._makeSegment()
logger.info("created first segment for new index '%s'" % self.name)
else:
logger.info("loaded %i segments for index '%s'" % (len(self._segments), self.name))
# get a reference to the current segment
self._current = self._segments[-1]
logger.debug("opened event index '%s' (%s)" % (self.name, str(self._indexUUID)))
except:
self.close()
raise
def __init__(self, env, name, fieldstore):
self.name = name
self._segments = []
self._indexSize = 0
self._currentSize = 0
self._lastModified = 0
self._lastId = EVID_MIN
backend.Index.__init__(self, env, name)
try:
# load schema
self._schema = Schema(self, fieldstore)
# load data segments
with self.new_txn() as txn:
for segmentId in self.iter_segments(txn):
segment = Segment(txn, self, segmentId)
last_update = segment.get_meta(txn, u'last-update')
self._currentSize = last_update[u'size']
self._indexSize += last_update[u'size']
lastId = last_update[u'last-id']
lastId = EVID(lastId[0], lastId[1])
if lastId > self._lastId:
self._lastId = last_update[u'last-id']
if last_update[u'last-modified'] > self._lastModified:
self._lastModified = last_update[u'last-modified']
self._segments.append(segment)
# if the index has no segments, create one
if self._segments == []:
with self.new_txn() as txn:
segmentId = self.new_segment(txn)
segment = Segment(txn, self, segmentId)
segment.set_meta(txn, u'created-on', int(time.time()))
last_update = {
u'size': self._indexSize,
u'last-id': [self._lastId.ts, self._lastId.offset],
u'last-modified': self._lastModified
}
segment.set_meta(txn, u'last-update', last_update)
self._segments.append(segment)
logger.info("created first segment for new index '%s'" % name)
else:
logger.info("found %i events in %i segments for index '%s'" % (
self._indexSize, len(self._segments), name))
logger.debug("last evid is %s" % self._lastId)
# get a reference to the current segment
self._current = self._segments[-1]
logger.debug("opened event index '%s'" % self.name)
except:
self.close()
raise
def _makeSegment(self):
with self.new_txn() as txn:
try:
segmentId = self.get_meta(txn, u'last-segment-id', RMW=True)
except KeyError:
segmentId = 0
segmentId = segmentId + 1
self.set_meta(txn, u'last-segment-id', segmentId)
segmentName = u"%s.%i" % (self.name, segmentId)
segmentUUID = unicode(uuid5(self._indexUUID, str(segmentName)))
self.set_segment(txn, segmentName, segmentUUID, NOOVERWRITE=True)
segment = Segment(self._env, txn, segmentName)
segment.set_meta(txn, u'created-on', int(time.time()))
segment.set_meta(txn, u'uuid', segmentUUID)
last_update = {
u'segment-size': 0,
u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
u'last-modified': 0
}
segment.set_meta(txn, u'last-update', last_update)
with self._segmentLock:
self._segments.append(segment)
self._current = segment
return segment
def _makeSegment(self):
with self.new_txn() as txn:
try:
segmentId = self.get_meta(txn, u'last-segment-id', RMW=True)
except KeyError:
segmentId = 0
segmentId = segmentId + 1
self.set_meta(txn, u'last-segment-id', segmentId)
segmentName = u"%s.%i" % (self.name, segmentId)
segmentUUID = unicode(uuid5(self._indexUUID, str(segmentName)))
self.set_segment(txn, segmentName, segmentUUID, NOOVERWRITE=True)
segment = Segment(self._env, txn, segmentName)
segment.set_meta(txn, u'created-on', int(time.time()))
segment.set_meta(txn, u'uuid', segmentUUID)
last_update = {
u'segment-size': 0,
u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
u'last-modified': 0
}
segment.set_meta(txn, u'last-update', last_update)
with self._segmentLock:
self._segments.append(segment)
self._current = segment
return segment
def __init__(self, output):
self.name = output._indexName
self._env = output._plugin._env
backend.Index.__init__(self, self._env, self.name)
self._segmentLock = Lock()
self._segments = []
self._current = None
self._fieldLock = Lock()
self._fieldstore = output._fieldstore
self._fields = {}
self._indexUUID = None
try:
# load index metadata
with self.new_txn() as txn:
try:
self._indexUUID = UUID(
self.get_meta(txn, u'uuid', RMW=True))
except KeyError:
self._indexUUID = uuid4()
self.set_meta(txn, u'uuid', unicode(self._indexUUID))
# load schema
with self.new_txn() as txn:
for fieldname, fieldspec in self.iter_fields(txn):
self._fields[fieldname] = pickle.loads(str(fieldspec))
# verify that the field type is consistent
for fieldtype, stored in self._fields[fieldname].items():
field = self._fieldstore.getField(fieldtype)
if not stored.field.__class__ == field.__class__:
raise SchemaError(
"schema field %s:%s does not match registered type %s"
% (fieldname, fieldtype,
field.__class__.__name__))
# load data segments
indexSize = 0
with self.new_txn() as txn:
for segmentName, segmentUUID in self.iter_segments(txn):
segment = Segment(self._env, txn, segmentName)
try:
foundUUID = segment.get_meta(txn, u'uuid')
except KeyError:
foundUUID = None
if foundUUID == None or segmentUUID != foundUUID:
logger.debug("index segment %s has invalid UUID %s" %
(segmentName, foundUUID))
segment.close()
else:
self._segments.append(segment)
logger.debug("opened index segment '%s'" % segmentName)
# if the index has no segments, create one
if self._segments == []:
self._makeSegment()
logger.info("created first segment for new index '%s'" %
self.name)
else:
logger.info("loaded %i segments for index '%s'" %
(len(self._segments), self.name))
# get a reference to the current segment
self._current = self._segments[-1]
logger.debug("opened event index '%s' (%s)" %
(self.name, str(self._indexUUID)))
except:
self.close()
raise