def create(self, forceIndexBuild=False, validateData=False): textFileTime = os.stat(self.__textFile)[8] # gdbm on OS X seems to voluntarily append .db, trying to catch that indexNeeded = forceIndexBuild for dbFile in ( self.__dbFile + os.path.extsep + 'db', self.__dbFile ): if os.path.exists(dbFile): if textFileTime < os.stat(dbFile)[8]: if indexNeeded: log.msg('Forced index rebuild %s' % dbFile) elif not whichdb(dbFile): indexNeeded = True log.msg('Unsupported index format, rebuilding index %s' % dbFile) else: indexNeeded = True log.msg('Index %s out of date' % dbFile) break else: indexNeeded = True log.msg('Index %s does not exist for data file %s' % (self.__dbFile, self.__textFile)) if indexNeeded: # these might speed-up indexing open_flags = 'nfu' while open_flags: try: db = dbm.open(self.__dbFile, open_flags) except Exception: open_flags = open_flags[:-1] continue else: break else: raise error.SnmpsimError('Failed to create %s for data file %s: %s' % (self.__dbFile, self.__textFile, sys.exc_info()[1])) try: text = open(self.__textFile, 'rb') except: raise error.SnmpsimError('Failed to open data file %s: %s' % (self.__dbFile, sys.exc_info()[0])) log.msg('Building index %s for data file %s (open flags \"%s\")...' % (self.__dbFile, self.__textFile, open_flags)) sys.stdout.flush() lineNo = 0 offset = 0 prevOffset = -1 while True: line, lineNo, offset = getRecord(text, lineNo, offset) if not line: # reference to last OID in data file db['last'] = '%d,%d,%d' % (offset, 0, prevOffset) break try: oid, tag, val = self.__textParser.grammar.parse(line) except Exception: db.close() exc = sys.exc_info()[1] try: os.remove(self.__dbFile) except OSError: pass raise error.SnmpsimError('Data error at %s:%d: %s' % (self.__textFile, lineNo, exc)) if validateData: try: self.__textParser.evaluateOid(oid) except Exception: db.close() exc = sys.exc_info()[1] try: os.remove(self.__dbFile) except OSError: pass raise error.SnmpsimError('OID error at %s:%d: %s' % (self.__textFile, lineNo, exc)) try: self.__textParser.evaluateValue( oid, tag, val, dataValidation=True ) except Exception: log.msg( 'ERROR at line %s, value %r: %s' % \ (lineNo, val, sys.exc_info()[1]) ) # for lines serving subtrees, type is empty in tag field db[oid] = '%d,%d,%d' % (offset, tag[0] == ':', prevOffset) if tag[0] == ':': prevOffset = offset else: prevOffset = -1 # not a subtree - no backreference offset += len(line) text.close() db.close() log.msg('...%d entries indexed' % lineNo) self.__textFileTime = os.stat(self.__textFile)[8] self.__dbType = whichdb(self.__dbFile) return self
def create(self, forceIndexBuild=False, validateData=False): textFileTime = os.stat(self.__textFile)[8] # gdbm on OS X seems to voluntarily append .db, trying to catch that indexNeeded = forceIndexBuild for dbFile in ( self.__dbFile + os.path.extsep + 'db', self.__dbFile ): if os.path.exists(dbFile): if textFileTime < os.stat(dbFile)[8]: if indexNeeded: log.msg('Forced index rebuild %s' % dbFile) elif not whichdb(dbFile): indexNeeded = True log.msg('Unsupported index format, rebuilding index %s' % dbFile) else: indexNeeded = True log.msg('Index %s out of date' % dbFile) break else: indexNeeded = True log.msg('Index %s does not exist for data file %s' % (self.__dbFile, self.__textFile)) if indexNeeded: # these might speed-up indexing open_flags = 'nfu' while open_flags: try: db = dbm.open(self.__dbFile, open_flags) except Exception: open_flags = open_flags[:-1] continue else: break else: raise error.SnmpsimError('Failed to create %s for data file %s: %s' % (self.__dbFile, self.__textFile, sys.exc_info()[1])) try: text = open(self.__textFile, 'rb') except: raise error.SnmpsimError('Failed to open data file %s: %s' % (self.__dbFile, sys.exc_info()[0])) log.msg('Building index %s for data file %s (open flags \"%s\")...' % (self.__dbFile, self.__textFile, open_flags)) sys.stdout.flush() lineNo = 0 offset = 0 prevOffset = -1 while 1: line, lineNo, offset = getRecord(text, lineNo, offset) if not line: # reference to last OID in data file db['last'] = '%d,%d,%d' % (offset, 0, prevOffset) break try: oid, tag, val = self.__textParser.grammar.parse(line) except Exception: db.close() exc = sys.exc_info()[1] try: os.remove(self.__dbFile) except OSError: pass raise error.SnmpsimError('Data error at %s:%d: %s' % (self.__textFile, lineNo, exc)) if validateData: try: self.__textParser.evaluateOid(oid) except Exception: db.close() exc = sys.exc_info()[1] try: os.remove(self.__dbFile) except OSError: pass raise error.SnmpsimError('OID error at %s:%d: %s' % (self.__textFile, lineNo, exc)) try: self.__textParser.evaluateValue( oid, tag, val, dataValidation=True ) except Exception: log.msg( 'ERROR at line %s, value %r: %s' % \ (lineNo, val, sys.exc_info()[1]) ) # for lines serving subtrees, type is empty in tag field db[oid] = '%d,%d,%d' % (offset, tag[0] == ':', prevOffset) if tag[0] == ':': prevOffset = offset else: prevOffset = -1 # not a subtree - no backreference offset += len(line) text.close() db.close() log.msg('...%d entries indexed' % lineNo) self.__textFileTime = os.stat(self.__textFile)[8] self.__dbType = whichdb(self.__dbFile) return self
def variate(oid, tag, value, **context): if 'settings' not in recordContext: recordContext['settings'] = dict( [split(x, '=') for x in split(value, ',')]) if 'dir' not in recordContext['settings']: log.msg('multiplex: snapshot directory not specified') return context['origOid'], tag, context['errorStatus'] recordContext['settings']['dir'] = recordContext['settings'][ 'dir'].replace('/', os.path.sep) if recordContext['settings']['dir'][0] != os.path.sep: for x in confdir.data: d = os.path.join(x, recordContext['settings']['dir']) if os.path.exists(d): break else: log.msg('multiplex: directory %s not found' % recordContext['settings']['dir']) return context['origOid'], tag, context['errorStatus'] else: d = recordContext['settings']['dir'] recordContext['dirmap'] = dict([ (int(os.path.basename(x).split(os.path.extsep)[0]), os.path.join(d, x)) for x in os.listdir(d) if x[-7:] == 'snmprec' ]) recordContext['keys'] = list(recordContext['dirmap'].keys()) recordContext['bounds'] = (min(recordContext['keys']), max(recordContext['keys'])) if 'period' in recordContext['settings']: recordContext['settings']['period'] = float( recordContext['settings']['period']) else: recordContext['settings']['period'] = 60.0 if 'wrap' in recordContext['settings']: recordContext['settings']['wrap'] = bool( recordContext['settings']['wrap']) else: recordContext['settings']['wrap'] = False if 'control' in recordContext['settings']: recordContext['settings']['control'] = rfc1902.ObjectName( recordContext['settings']['control']) log.msg( 'multiplex: using control OID %s for subtree %s, time-based multiplexing disabled' % (recordContext['settings']['control'], oid)) recordContext['ready'] = True if 'ready' not in recordContext: return context['origOid'], tag, context['errorStatus'] if oid not in moduleContext: moduleContext[oid] = {} if context['setFlag']: if 'control' in recordContext['settings'] and \ recordContext['settings']['control'] == context['origOid']: fileno = int(context['origValue']) if fileno >= len(recordContext['keys']): log.msg('multiplex: .snmprec file number %s over limit of %s' % (fileno, len(recordContext['keys']))) return context['origOid'], tag, context['errorStatus'] moduleContext[oid]['fileno'] = fileno log.msg('multiplex: switched to file #%s (%s)' % (recordContext['keys'][fileno], recordContext['dirmap'][recordContext['keys'][fileno]])) return context['origOid'], tag, context['origValue'] else: return context['origOid'], tag, context['errorStatus'] if 'control' in recordContext['settings']: if 'fileno' not in moduleContext[oid]: moduleContext[oid]['fileno'] = 0 if (not context['nextFlag'] and recordContext['settings']['control'] == context['origOid']): return context['origOid'], tag, rfc1902.Integer32( moduleContext[oid]['fileno']) else: timeslot = (time.time() - moduleContext['booted']) % ( recordContext['settings']['period'] * len(recordContext['dirmap'])) fileslot = int( timeslot / recordContext['settings']['period']) + recordContext['bounds'][0] fileno = bisect.bisect(recordContext['keys'], fileslot) - 1 if ('fileno' not in moduleContext[oid] or moduleContext[oid]['fileno'] < fileno or recordContext['settings']['wrap']): moduleContext[oid]['fileno'] = fileno datafile = recordContext['dirmap'][recordContext['keys'][moduleContext[oid] ['fileno']]] if ('datafile' not in moduleContext[oid] or moduleContext[oid]['datafile'] != datafile): if 'datafileobj' in moduleContext[oid]: moduleContext[oid]['datafileobj'].close() moduleContext[oid]['datafileobj'] = RecordIndex( datafile, SnmprecRecord()).create() moduleContext[oid]['datafile'] = datafile log.msg('multiplex: switching to data file %s for %s' % (datafile, context['origOid'])) text, db = moduleContext[oid]['datafileobj'].getHandles() textOid = str( rfc1902.OctetString('.'.join(['%s' % x for x in context['origOid']]))) try: line = moduleContext[oid]['datafileobj'].lookup(textOid) except KeyError: offset = searchRecordByOid(context['origOid'], text, SnmprecRecord()) exactMatch = False else: offset, subtreeFlag, prevOffset = line.split(str2octs(',')) exactMatch = True text.seek(int(offset)) line, _, _ = getRecord(text) # matched line if context['nextFlag']: if exactMatch: line, _, _ = getRecord(text) else: if not exactMatch: return context['origOid'], tag, context['errorStatus'] if not line: return context['origOid'], tag, context['errorStatus'] try: oid, value = SnmprecRecord().evaluate(line) except error.SnmpsimError: oid, value = context['origOid'], context['errorStatus'] return oid, tag, value
compiler.addMibCompiler( mibBuilder, sources=mibSources or defaultMibSources ) if isinstance(startOID, rfc1902.ObjectIdentity): startOID.resolveWithMib(mibViewController) if isinstance(stopOID, rfc1902.ObjectIdentity): stopOID.resolveWithMib(mibViewController) recordsList = [] for inputFile in inputFiles: if verboseFlag: sys.stderr.write('# Input file #%s, processing records from %s till %s\r\n' % (inputFiles.index(inputFile), startOID or 'the beginning', stopOID or 'the end')) lineNo = 0 while True: line, recLineNo, _ = getRecord(inputFile, lineNo) if not line: break if recLineNo != lineNo + 1: if verboseFlag: sys.stderr.write('# Losing comment at lines %s..%s (input file #%s)\r\n' % (lineNo + 1, recLineNo - 1, inputFiles.index(inputFile))) lineNo = recLineNo lostComments += 1 backdoor = {} try: oid, value = recordsSet[srcRecordType].evaluate(line, backdoor=backdoor) except error.SnmpsimError: if ignoreBrokenRecords: if verboseFlag: sys.stderr.write('# Skipping broken record <%s>: %s\r\n' % (line, sys.exc_info()[1])) brokenCount += 1
compiler.addMibCompiler( mibBuilder, sources=mibSources or defaultMibSources ) if isinstance(startOID, rfc1902.ObjectIdentity): startOID.resolveWithMib(mibViewController) if isinstance(stopOID, rfc1902.ObjectIdentity): stopOID.resolveWithMib(mibViewController) recordsList = [] for inputFile in inputFiles: if verboseFlag: sys.stderr.write('# Input file #%s, processing records from %s till %s\r\n' % (inputFiles.index(inputFile), startOID or 'the beginning', stopOID or 'the end')) lineNo = 0 while True: line, recLineNo, _ = getRecord(inputFile, lineNo) if not line: break if recLineNo != lineNo+1: if verboseFlag: sys.stderr.write('# Losing comment at lines %s..%s (input file #%s)\r\n' % (lineNo+1, recLineNo-1, inputFiles.index(inputFile))) lineNo = recLineNo lostComments += 1 backdoor = {} try: oid, value = recordsSet[srcRecordType].evaluate(line, backdoor=backdoor) except error.SnmpsimError: if ignoreBrokenRecords: if verboseFlag: sys.stderr.write('# Skipping broken record <%s>\r\n' % line) brokenCount += 1
def variate(oid, tag, value, **context): if 'settings' not in recordContext: recordContext['settings'] = dict( [split(x, '=') for x in split(value, ',')]) if 'dir' not in recordContext['settings']: log.msg('multiplex: snapshot directory not specified') return context['origOid'], tag, context['errorStatus'] recordContext['settings']['dir'] = recordContext[ 'settings']['dir'].replace('/', os.path.sep) if recordContext['settings']['dir'][0] != os.path.sep: for x in confdir.data: d = os.path.join(x, recordContext['settings']['dir']) if os.path.exists(d): break else: log.msg('multiplex: directory %s not ' 'found' % recordContext['settings']['dir']) return context['origOid'], tag, context['errorStatus'] else: d = recordContext['settings']['dir'] recordContext['dirmap'] = dict( [(int(os.path.basename(x).split(os.path.extsep)[0]), os.path.join(d, x)) for x in os.listdir(d) if x[-7:] == 'snmprec'] ) recordContext['keys'] = list(recordContext['dirmap']) recordContext['bounds'] = ( min(recordContext['keys']), max(recordContext['keys'])) if 'period' in recordContext['settings']: recordContext['settings']['period'] = float( recordContext['settings']['period']) else: recordContext['settings']['period'] = 60.0 if 'wrap' in recordContext['settings']: recordContext['settings']['wrap'] = bool( recordContext['settings']['wrap']) else: recordContext['settings']['wrap'] = False if 'control' in recordContext['settings']: recordContext['settings']['control'] = rfc1902.ObjectName( recordContext['settings']['control']) log.msg( 'multiplex: using control OID %s for subtree %s, ' 'time-based multiplexing ' 'disabled' % (recordContext['settings']['control'], oid)) recordContext['ready'] = True if 'ready' not in recordContext: return context['origOid'], tag, context['errorStatus'] if oid not in moduleContext: moduleContext[oid] = {} if context['setFlag']: if 'control' in ( recordContext['settings'] and recordContext['settings']['control'] == context['origOid']): fileno = int(context['origValue']) if fileno >= len(recordContext['keys']): log.msg('multiplex: .snmprec file number %s over limit of' ' %s' % (fileno, len(recordContext['keys']))) return context['origOid'], tag, context['errorStatus'] moduleContext[oid]['fileno'] = fileno log.msg( 'multiplex: switched to file #%s ' '(%s)' % (recordContext['keys'][fileno], recordContext['dirmap'][recordContext['keys'][fileno]])) return context['origOid'], tag, context['origValue'] else: return context['origOid'], tag, context['errorStatus'] if 'control' in recordContext['settings']: if 'fileno' not in moduleContext[oid]: moduleContext[oid]['fileno'] = 0 if (not context['nextFlag'] and recordContext['settings']['control'] == context['origOid']): val = rfc1902.Integer32(moduleContext[oid]['fileno']) return context['origOid'], tag, val else: period = recordContext['settings']['period'] uptime = time.time() - moduleContext['booted'] timeslot = uptime % (period * len(recordContext['dirmap'])) fileslot = int(timeslot / period) + recordContext['bounds'][0] fileno = bisect.bisect(recordContext['keys'], fileslot) - 1 if ('fileno' not in moduleContext[oid] or moduleContext[oid]['fileno'] < fileno or recordContext['settings']['wrap']): moduleContext[oid]['fileno'] = fileno datafile = recordContext['dirmap'][ recordContext['keys'][moduleContext[oid]['fileno']]] if ('datafile' not in moduleContext[oid] or moduleContext[oid]['datafile'] != datafile): if 'datafileobj' in moduleContext[oid]: moduleContext[oid]['datafileobj'].close() recordIndex = RecordIndex(datafile, SnmprecRecord()).create() moduleContext[oid]['datafileobj'] = recordIndex moduleContext[oid]['datafile'] = datafile log.msg( 'multiplex: switching to data file %s for ' '%s' % (datafile, context['origOid'])) text, db = moduleContext[oid]['datafileobj'].getHandles() textOid = str(rfc1902.OctetString( '.'.join(['%s' % x for x in context['origOid']]))) try: line = moduleContext[oid]['datafileobj'].lookup(textOid) except KeyError: offset = searchRecordByOid(context['origOid'], text, SnmprecRecord()) exactMatch = False else: offset, subtreeFlag, prevOffset = line.split(str2octs(',')) exactMatch = True text.seek(int(offset)) line, _, _ = getRecord(text) # matched line if context['nextFlag']: if exactMatch: line, _, _ = getRecord(text) else: if not exactMatch: return context['origOid'], tag, context['errorStatus'] if not line: return context['origOid'], tag, context['errorStatus'] try: oid, value = SnmprecRecord().evaluate(line) except error.SnmpsimError: oid, value = context['origOid'], context['errorStatus'] return oid, tag, value