Beispiel #1
0
 def rotateSegments(self, segRotation, segRetention):
     """
     Allocate a new Segment, making it the new current segment.
     """
     # if the current segment contains more events than specified by
     # segRotation, then rotate the index to generate a new segment.
     if segRotation > 0 and self._currentSize >= segRotation:
         with self.new_txn() as txn:
             segmentId = self.new_segment(txn)
             segment = Segment(txn, self, segmentId)
             segment.set_meta(txn, u'created-on', int(time.time()))
             last_update = {
                 u'size': 0,
                 u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
                 u'last-modified': 0
                 }
             segment.set_meta(txn, u'last-update', last_update)
         self._segments.append(segment)
         self._current = segment
         self._currentSize = 0
         logger.debug("rotated current segment, new segment is %s" % segment.fullName)
         # if the index contains more segments than specified by segRetention,
         # then delete the oldest segment.
         if segRetention > 0:
             if len(self._segments) > segRetention:
                 for segment in self._segments[0:len(self._segments)-segRetention]:
                     self.delete(segment)
Beispiel #2
0
 def __init__(self, output):
     self.name = output._indexName
     self._env = output._plugin._env
     backend.Index.__init__(self, self._env, self.name)
     self._segmentLock = Lock()
     self._segments = []
     self._current = None
     self._fieldLock = Lock()
     self._fieldstore = output._fieldstore
     self._fields = {}
     self._indexUUID = None
     try:
         # load index metadata
         with self.new_txn() as txn:
             try:
                 self._indexUUID = UUID(self.get_meta(txn, u'uuid', RMW=True))
             except KeyError:
                 self._indexUUID = uuid4()
                 self.set_meta(txn, u'uuid', unicode(self._indexUUID))
         # load schema
         with self.new_txn() as txn:
             for fieldname,fieldspec in self.iter_fields(txn):
                 self._fields[fieldname] = pickle.loads(str(fieldspec))
                 # verify that the field type is consistent
                 for fieldtype,stored in self._fields[fieldname].items():
                     field = self._fieldstore.getField(fieldtype)
                     if not stored.field.__class__ == field.__class__:
                         raise SchemaError("schema field %s:%s does not match registered type %s" % (
                             fieldname, fieldtype, field.__class__.__name__))
         # load data segments
         indexSize = 0
         with self.new_txn() as txn:
             for segmentName,segmentUUID in self.iter_segments(txn):
                 segment = Segment(self._env, txn, segmentName)
                 try:
                     foundUUID = segment.get_meta(txn, u'uuid')
                 except KeyError:
                     foundUUID = None
                 if foundUUID == None or segmentUUID != foundUUID:
                     logger.debug("index segment %s has invalid UUID %s" % (segmentName, foundUUID))
                     segment.close()
                 else:
                     self._segments.append(segment)
                     logger.debug("opened index segment '%s'" % segmentName)
         # if the index has no segments, create one
         if self._segments == []:
             self._makeSegment()
             logger.info("created first segment for new index '%s'" % self.name)
         else:
             logger.info("loaded %i segments for index '%s'" % (len(self._segments), self.name))
         # get a reference to the current segment
         self._current = self._segments[-1]
         logger.debug("opened event index '%s' (%s)" % (self.name, str(self._indexUUID)))
     except:
         self.close()
         raise
Beispiel #3
0
 def __init__(self, env, name, fieldstore):
     self.name = name
     self._segments = []
     self._indexSize = 0
     self._currentSize = 0
     self._lastModified = 0
     self._lastId = EVID_MIN
     backend.Index.__init__(self, env, name)
     try:
         # load schema
         self._schema = Schema(self, fieldstore)
         # load data segments
         with self.new_txn() as txn:
             for segmentId in self.iter_segments(txn):
                 segment = Segment(txn, self, segmentId)
                 last_update = segment.get_meta(txn, u'last-update')
                 self._currentSize = last_update[u'size']
                 self._indexSize += last_update[u'size']
                 lastId = last_update[u'last-id']
                 lastId = EVID(lastId[0], lastId[1])
                 if lastId > self._lastId:
                     self._lastId = last_update[u'last-id']
                 if last_update[u'last-modified'] > self._lastModified:
                     self._lastModified = last_update[u'last-modified']
                 self._segments.append(segment)
         # if the index has no segments, create one
         if self._segments == []:
             with self.new_txn() as txn:
                 segmentId = self.new_segment(txn)
                 segment = Segment(txn, self, segmentId)
                 segment.set_meta(txn, u'created-on', int(time.time()))
                 last_update = {
                     u'size': self._indexSize,
                     u'last-id': [self._lastId.ts, self._lastId.offset],
                     u'last-modified': self._lastModified
                     }
                 segment.set_meta(txn, u'last-update', last_update)
             self._segments.append(segment)
             logger.info("created first segment for new index '%s'" % name)
         else:
             logger.info("found %i events in %i segments for index '%s'" % (
                 self._indexSize, len(self._segments), name))
         logger.debug("last evid is %s" % self._lastId)
         # get a reference to the current segment
         self._current = self._segments[-1]
         logger.debug("opened event index '%s'" % self.name)
     except:
         self.close()
         raise
Beispiel #4
0
 def _makeSegment(self):
     with self.new_txn() as txn:
         try:
             segmentId = self.get_meta(txn, u'last-segment-id', RMW=True)
         except KeyError:
             segmentId = 0
         segmentId = segmentId + 1
         self.set_meta(txn, u'last-segment-id', segmentId)
         segmentName = u"%s.%i" % (self.name, segmentId)
         segmentUUID = unicode(uuid5(self._indexUUID, str(segmentName)))
         self.set_segment(txn, segmentName, segmentUUID, NOOVERWRITE=True)
         segment = Segment(self._env, txn, segmentName)
         segment.set_meta(txn, u'created-on', int(time.time()))
         segment.set_meta(txn, u'uuid', segmentUUID)
         last_update = {
             u'segment-size': 0,
             u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
             u'last-modified': 0
         }
         segment.set_meta(txn, u'last-update', last_update)
     with self._segmentLock:
         self._segments.append(segment)
         self._current = segment
     return segment
Beispiel #5
0
 def _makeSegment(self):
     with self.new_txn() as txn:
         try:
             segmentId = self.get_meta(txn, u'last-segment-id', RMW=True)
         except KeyError:
             segmentId = 0
         segmentId = segmentId + 1
         self.set_meta(txn, u'last-segment-id', segmentId)
         segmentName = u"%s.%i" % (self.name, segmentId)
         segmentUUID = unicode(uuid5(self._indexUUID, str(segmentName)))
         self.set_segment(txn, segmentName, segmentUUID, NOOVERWRITE=True)
         segment = Segment(self._env, txn, segmentName)
         segment.set_meta(txn, u'created-on', int(time.time()))
         segment.set_meta(txn, u'uuid', segmentUUID)
         last_update = {
             u'segment-size': 0,
             u'last-id': [EVID_MIN.ts, EVID_MIN.offset],
             u'last-modified': 0
             }
         segment.set_meta(txn, u'last-update', last_update)
     with self._segmentLock:
         self._segments.append(segment)
         self._current = segment
     return segment
Beispiel #6
0
 def __init__(self, output):
     self.name = output._indexName
     self._env = output._plugin._env
     backend.Index.__init__(self, self._env, self.name)
     self._segmentLock = Lock()
     self._segments = []
     self._current = None
     self._fieldLock = Lock()
     self._fieldstore = output._fieldstore
     self._fields = {}
     self._indexUUID = None
     try:
         # load index metadata
         with self.new_txn() as txn:
             try:
                 self._indexUUID = UUID(
                     self.get_meta(txn, u'uuid', RMW=True))
             except KeyError:
                 self._indexUUID = uuid4()
                 self.set_meta(txn, u'uuid', unicode(self._indexUUID))
         # load schema
         with self.new_txn() as txn:
             for fieldname, fieldspec in self.iter_fields(txn):
                 self._fields[fieldname] = pickle.loads(str(fieldspec))
                 # verify that the field type is consistent
                 for fieldtype, stored in self._fields[fieldname].items():
                     field = self._fieldstore.getField(fieldtype)
                     if not stored.field.__class__ == field.__class__:
                         raise SchemaError(
                             "schema field %s:%s does not match registered type %s"
                             % (fieldname, fieldtype,
                                field.__class__.__name__))
         # load data segments
         indexSize = 0
         with self.new_txn() as txn:
             for segmentName, segmentUUID in self.iter_segments(txn):
                 segment = Segment(self._env, txn, segmentName)
                 try:
                     foundUUID = segment.get_meta(txn, u'uuid')
                 except KeyError:
                     foundUUID = None
                 if foundUUID == None or segmentUUID != foundUUID:
                     logger.debug("index segment %s has invalid UUID %s" %
                                  (segmentName, foundUUID))
                     segment.close()
                 else:
                     self._segments.append(segment)
                     logger.debug("opened index segment '%s'" % segmentName)
         # if the index has no segments, create one
         if self._segments == []:
             self._makeSegment()
             logger.info("created first segment for new index '%s'" %
                         self.name)
         else:
             logger.info("loaded %i segments for index '%s'" %
                         (len(self._segments), self.name))
         # get a reference to the current segment
         self._current = self._segments[-1]
         logger.debug("opened event index '%s' (%s)" %
                      (self.name, str(self._indexUUID)))
     except:
         self.close()
         raise