def create(self, forceIndexBuild=False, validateData=False):
textFileTime = os.stat(self.__textFile)[8]
# gdbm on OS X seems to voluntarily append .db, trying to catch that
indexNeeded = forceIndexBuild
for dbFile in (
self.__dbFile + os.path.extsep + 'db',
self.__dbFile
):
if os.path.exists(dbFile):
if textFileTime < os.stat(dbFile)[8]:
if indexNeeded:
log.msg('Forced index rebuild %s' % dbFile)
elif not whichdb(dbFile):
indexNeeded = True
log.msg('Unsupported index format, rebuilding index %s' % dbFile)
else:
indexNeeded = True
log.msg('Index %s out of date' % dbFile)
break
else:
indexNeeded = True
log.msg('Index %s does not exist for data file %s' % (self.__dbFile, self.__textFile))
if indexNeeded:
# these might speed-up indexing
open_flags = 'nfu'
while open_flags:
try:
db = dbm.open(self.__dbFile, open_flags)
except Exception:
open_flags = open_flags[:-1]
continue
else:
break
else:
raise error.SnmpsimError('Failed to create %s for data file %s: %s' % (self.__dbFile, self.__textFile, sys.exc_info()[1]))
try:
text = open(self.__textFile, 'rb')
except:
raise error.SnmpsimError('Failed to open data file %s: %s' % (self.__dbFile, sys.exc_info()[0]))
log.msg('Building index %s for data file %s (open flags \"%s\")...' % (self.__dbFile, self.__textFile, open_flags))
sys.stdout.flush()
lineNo = 0
offset = 0
prevOffset = -1
while True:
line, lineNo, offset = getRecord(text, lineNo, offset)
if not line:
# reference to last OID in data file
db['last'] = '%d,%d,%d' % (offset, 0, prevOffset)
break
try:
oid, tag, val = self.__textParser.grammar.parse(line)
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFile)
except OSError:
pass
raise error.SnmpsimError('Data error at %s:%d: %s' % (self.__textFile, lineNo, exc))
if validateData:
try:
self.__textParser.evaluateOid(oid)
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFile)
except OSError:
pass
raise error.SnmpsimError('OID error at %s:%d: %s' % (self.__textFile, lineNo, exc))
try:
self.__textParser.evaluateValue(
oid, tag, val, dataValidation=True
)
except Exception:
log.msg(
'ERROR at line %s, value %r: %s' % \
(lineNo, val, sys.exc_info()[1])
)
# for lines serving subtrees, type is empty in tag field
db[oid] = '%d,%d,%d' % (offset, tag[0] == ':', prevOffset)
if tag[0] == ':':
prevOffset = offset
else:
prevOffset = -1 # not a subtree - no backreference
offset += len(line)
text.close()
db.close()
log.msg('...%d entries indexed' % lineNo)
self.__textFileTime = os.stat(self.__textFile)[8]
self.__dbType = whichdb(self.__dbFile)
return self
def create(self, forceIndexBuild=False, validateData=False):
textFileTime = os.stat(self.__textFile)[8]
# gdbm on OS X seems to voluntarily append .db, trying to catch that
indexNeeded = forceIndexBuild
for dbFile in (
self.__dbFile + os.path.extsep + 'db',
self.__dbFile
):
if os.path.exists(dbFile):
if textFileTime < os.stat(dbFile)[8]:
if indexNeeded:
log.msg('Forced index rebuild %s' % dbFile)
elif not whichdb(dbFile):
indexNeeded = True
log.msg('Unsupported index format, rebuilding index %s' % dbFile)
else:
indexNeeded = True
log.msg('Index %s out of date' % dbFile)
break
else:
indexNeeded = True
log.msg('Index %s does not exist for data file %s' % (self.__dbFile, self.__textFile))
if indexNeeded:
# these might speed-up indexing
open_flags = 'nfu'
while open_flags:
try:
db = dbm.open(self.__dbFile, open_flags)
except Exception:
open_flags = open_flags[:-1]
continue
else:
break
else:
raise error.SnmpsimError('Failed to create %s for data file %s: %s' % (self.__dbFile, self.__textFile, sys.exc_info()[1]))
try:
text = open(self.__textFile, 'rb')
except:
raise error.SnmpsimError('Failed to open data file %s: %s' % (self.__dbFile, sys.exc_info()[0]))
log.msg('Building index %s for data file %s (open flags \"%s\")...' % (self.__dbFile, self.__textFile, open_flags))
sys.stdout.flush()
lineNo = 0
offset = 0
prevOffset = -1
while 1:
line, lineNo, offset = getRecord(text, lineNo, offset)
if not line:
# reference to last OID in data file
db['last'] = '%d,%d,%d' % (offset, 0, prevOffset)
break
try:
oid, tag, val = self.__textParser.grammar.parse(line)
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFile)
except OSError:
pass
raise error.SnmpsimError('Data error at %s:%d: %s' % (self.__textFile, lineNo, exc))
if validateData:
try:
self.__textParser.evaluateOid(oid)
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFile)
except OSError:
pass
raise error.SnmpsimError('OID error at %s:%d: %s' % (self.__textFile, lineNo, exc))
try:
self.__textParser.evaluateValue(
oid, tag, val, dataValidation=True
)
except Exception:
log.msg(
'ERROR at line %s, value %r: %s' % \
(lineNo, val, sys.exc_info()[1])
)
# for lines serving subtrees, type is empty in tag field
db[oid] = '%d,%d,%d' % (offset, tag[0] == ':', prevOffset)
if tag[0] == ':':
prevOffset = offset
else:
prevOffset = -1 # not a subtree - no backreference
offset += len(line)
text.close()
db.close()
log.msg('...%d entries indexed' % lineNo)
self.__textFileTime = os.stat(self.__textFile)[8]
self.__dbType = whichdb(self.__dbFile)
return self
def variate(oid, tag, value, **context):
if 'settings' not in recordContext:
recordContext['settings'] = dict(
[split(x, '=') for x in split(value, ',')])
if 'dir' not in recordContext['settings']:
log.msg('multiplex: snapshot directory not specified')
return context['origOid'], tag, context['errorStatus']
recordContext['settings']['dir'] = recordContext['settings'][
'dir'].replace('/', os.path.sep)
if recordContext['settings']['dir'][0] != os.path.sep:
for x in confdir.data:
d = os.path.join(x, recordContext['settings']['dir'])
if os.path.exists(d):
break
else:
log.msg('multiplex: directory %s not found' %
recordContext['settings']['dir'])
return context['origOid'], tag, context['errorStatus']
else:
d = recordContext['settings']['dir']
recordContext['dirmap'] = dict([
(int(os.path.basename(x).split(os.path.extsep)[0]),
os.path.join(d, x)) for x in os.listdir(d) if x[-7:] == 'snmprec'
])
recordContext['keys'] = list(recordContext['dirmap'].keys())
recordContext['bounds'] = (min(recordContext['keys']),
max(recordContext['keys']))
if 'period' in recordContext['settings']:
recordContext['settings']['period'] = float(
recordContext['settings']['period'])
else:
recordContext['settings']['period'] = 60.0
if 'wrap' in recordContext['settings']:
recordContext['settings']['wrap'] = bool(
recordContext['settings']['wrap'])
else:
recordContext['settings']['wrap'] = False
if 'control' in recordContext['settings']:
recordContext['settings']['control'] = rfc1902.ObjectName(
recordContext['settings']['control'])
log.msg(
'multiplex: using control OID %s for subtree %s, time-based multiplexing disabled'
% (recordContext['settings']['control'], oid))
recordContext['ready'] = True
if 'ready' not in recordContext:
return context['origOid'], tag, context['errorStatus']
if oid not in moduleContext:
moduleContext[oid] = {}
if context['setFlag']:
if 'control' in recordContext['settings'] and \
recordContext['settings']['control'] == context['origOid']:
fileno = int(context['origValue'])
if fileno >= len(recordContext['keys']):
log.msg('multiplex: .snmprec file number %s over limit of %s' %
(fileno, len(recordContext['keys'])))
return context['origOid'], tag, context['errorStatus']
moduleContext[oid]['fileno'] = fileno
log.msg('multiplex: switched to file #%s (%s)' %
(recordContext['keys'][fileno],
recordContext['dirmap'][recordContext['keys'][fileno]]))
return context['origOid'], tag, context['origValue']
else:
return context['origOid'], tag, context['errorStatus']
if 'control' in recordContext['settings']:
if 'fileno' not in moduleContext[oid]:
moduleContext[oid]['fileno'] = 0
if (not context['nextFlag'] and recordContext['settings']['control']
== context['origOid']):
return context['origOid'], tag, rfc1902.Integer32(
moduleContext[oid]['fileno'])
else:
timeslot = (time.time() - moduleContext['booted']) % (
recordContext['settings']['period'] * len(recordContext['dirmap']))
fileslot = int(
timeslot /
recordContext['settings']['period']) + recordContext['bounds'][0]
fileno = bisect.bisect(recordContext['keys'], fileslot) - 1
if ('fileno' not in moduleContext[oid]
or moduleContext[oid]['fileno'] < fileno
or recordContext['settings']['wrap']):
moduleContext[oid]['fileno'] = fileno
datafile = recordContext['dirmap'][recordContext['keys'][moduleContext[oid]
['fileno']]]
if ('datafile' not in moduleContext[oid]
or moduleContext[oid]['datafile'] != datafile):
if 'datafileobj' in moduleContext[oid]:
moduleContext[oid]['datafileobj'].close()
moduleContext[oid]['datafileobj'] = RecordIndex(
datafile, SnmprecRecord()).create()
moduleContext[oid]['datafile'] = datafile
log.msg('multiplex: switching to data file %s for %s' %
(datafile, context['origOid']))
text, db = moduleContext[oid]['datafileobj'].getHandles()
textOid = str(
rfc1902.OctetString('.'.join(['%s' % x for x in context['origOid']])))
try:
line = moduleContext[oid]['datafileobj'].lookup(textOid)
except KeyError:
offset = searchRecordByOid(context['origOid'], text, SnmprecRecord())
exactMatch = False
else:
offset, subtreeFlag, prevOffset = line.split(str2octs(','))
exactMatch = True
text.seek(int(offset))
line, _, _ = getRecord(text) # matched line
if context['nextFlag']:
if exactMatch:
line, _, _ = getRecord(text)
else:
if not exactMatch:
return context['origOid'], tag, context['errorStatus']
if not line:
return context['origOid'], tag, context['errorStatus']
try:
oid, value = SnmprecRecord().evaluate(line)
except error.SnmpsimError:
oid, value = context['origOid'], context['errorStatus']
return oid, tag, value
compiler.addMibCompiler(
mibBuilder, sources=mibSources or defaultMibSources
)
if isinstance(startOID, rfc1902.ObjectIdentity):
startOID.resolveWithMib(mibViewController)
if isinstance(stopOID, rfc1902.ObjectIdentity):
stopOID.resolveWithMib(mibViewController)
recordsList = []
for inputFile in inputFiles:
if verboseFlag:
sys.stderr.write('# Input file #%s, processing records from %s till %s\r\n' % (inputFiles.index(inputFile), startOID or 'the beginning', stopOID or 'the end'))
lineNo = 0
while True:
line, recLineNo, _ = getRecord(inputFile, lineNo)
if not line:
break
if recLineNo != lineNo + 1:
if verboseFlag:
sys.stderr.write('# Losing comment at lines %s..%s (input file #%s)\r\n' % (lineNo + 1, recLineNo - 1, inputFiles.index(inputFile)))
lineNo = recLineNo
lostComments += 1
backdoor = {}
try:
oid, value = recordsSet[srcRecordType].evaluate(line, backdoor=backdoor)
except error.SnmpsimError:
if ignoreBrokenRecords:
if verboseFlag:
sys.stderr.write('# Skipping broken record <%s>: %s\r\n' % (line, sys.exc_info()[1]))
brokenCount += 1
compiler.addMibCompiler(
mibBuilder, sources=mibSources or defaultMibSources
)
if isinstance(startOID, rfc1902.ObjectIdentity):
startOID.resolveWithMib(mibViewController)
if isinstance(stopOID, rfc1902.ObjectIdentity):
stopOID.resolveWithMib(mibViewController)
recordsList = []
for inputFile in inputFiles:
if verboseFlag:
sys.stderr.write('# Input file #%s, processing records from %s till %s\r\n' % (inputFiles.index(inputFile), startOID or 'the beginning', stopOID or 'the end'))
lineNo = 0
while True:
line, recLineNo, _ = getRecord(inputFile, lineNo)
if not line:
break
if recLineNo != lineNo+1:
if verboseFlag:
sys.stderr.write('# Losing comment at lines %s..%s (input file #%s)\r\n' % (lineNo+1, recLineNo-1, inputFiles.index(inputFile)))
lineNo = recLineNo
lostComments += 1
backdoor = {}
try:
oid, value = recordsSet[srcRecordType].evaluate(line, backdoor=backdoor)
except error.SnmpsimError:
if ignoreBrokenRecords:
if verboseFlag:
sys.stderr.write('# Skipping broken record <%s>\r\n' % line)
brokenCount += 1
def variate(oid, tag, value, **context):
if 'settings' not in recordContext:
recordContext['settings'] = dict(
[split(x, '=') for x in split(value, ',')])
if 'dir' not in recordContext['settings']:
log.msg('multiplex: snapshot directory not specified')
return context['origOid'], tag, context['errorStatus']
recordContext['settings']['dir'] = recordContext[
'settings']['dir'].replace('/', os.path.sep)
if recordContext['settings']['dir'][0] != os.path.sep:
for x in confdir.data:
d = os.path.join(x, recordContext['settings']['dir'])
if os.path.exists(d):
break
else:
log.msg('multiplex: directory %s not '
'found' % recordContext['settings']['dir'])
return context['origOid'], tag, context['errorStatus']
else:
d = recordContext['settings']['dir']
recordContext['dirmap'] = dict(
[(int(os.path.basename(x).split(os.path.extsep)[0]),
os.path.join(d, x))
for x in os.listdir(d)
if x[-7:] == 'snmprec']
)
recordContext['keys'] = list(recordContext['dirmap'])
recordContext['bounds'] = (
min(recordContext['keys']), max(recordContext['keys']))
if 'period' in recordContext['settings']:
recordContext['settings']['period'] = float(
recordContext['settings']['period'])
else:
recordContext['settings']['period'] = 60.0
if 'wrap' in recordContext['settings']:
recordContext['settings']['wrap'] = bool(
recordContext['settings']['wrap'])
else:
recordContext['settings']['wrap'] = False
if 'control' in recordContext['settings']:
recordContext['settings']['control'] = rfc1902.ObjectName(
recordContext['settings']['control'])
log.msg(
'multiplex: using control OID %s for subtree %s, '
'time-based multiplexing '
'disabled' % (recordContext['settings']['control'], oid))
recordContext['ready'] = True
if 'ready' not in recordContext:
return context['origOid'], tag, context['errorStatus']
if oid not in moduleContext:
moduleContext[oid] = {}
if context['setFlag']:
if 'control' in (
recordContext['settings'] and
recordContext['settings']['control'] == context['origOid']):
fileno = int(context['origValue'])
if fileno >= len(recordContext['keys']):
log.msg('multiplex: .snmprec file number %s over limit of'
' %s' % (fileno, len(recordContext['keys'])))
return context['origOid'], tag, context['errorStatus']
moduleContext[oid]['fileno'] = fileno
log.msg(
'multiplex: switched to file #%s '
'(%s)' % (recordContext['keys'][fileno],
recordContext['dirmap'][recordContext['keys'][fileno]]))
return context['origOid'], tag, context['origValue']
else:
return context['origOid'], tag, context['errorStatus']
if 'control' in recordContext['settings']:
if 'fileno' not in moduleContext[oid]:
moduleContext[oid]['fileno'] = 0
if (not context['nextFlag'] and
recordContext['settings']['control'] == context['origOid']):
val = rfc1902.Integer32(moduleContext[oid]['fileno'])
return context['origOid'], tag, val
else:
period = recordContext['settings']['period']
uptime = time.time() - moduleContext['booted']
timeslot = uptime % (period * len(recordContext['dirmap']))
fileslot = int(timeslot / period) + recordContext['bounds'][0]
fileno = bisect.bisect(recordContext['keys'], fileslot) - 1
if ('fileno' not in moduleContext[oid] or
moduleContext[oid]['fileno'] < fileno or
recordContext['settings']['wrap']):
moduleContext[oid]['fileno'] = fileno
datafile = recordContext['dirmap'][
recordContext['keys'][moduleContext[oid]['fileno']]]
if ('datafile' not in moduleContext[oid] or
moduleContext[oid]['datafile'] != datafile):
if 'datafileobj' in moduleContext[oid]:
moduleContext[oid]['datafileobj'].close()
recordIndex = RecordIndex(datafile, SnmprecRecord()).create()
moduleContext[oid]['datafileobj'] = recordIndex
moduleContext[oid]['datafile'] = datafile
log.msg(
'multiplex: switching to data file %s for '
'%s' % (datafile, context['origOid']))
text, db = moduleContext[oid]['datafileobj'].getHandles()
textOid = str(rfc1902.OctetString(
'.'.join(['%s' % x for x in context['origOid']])))
try:
line = moduleContext[oid]['datafileobj'].lookup(textOid)
except KeyError:
offset = searchRecordByOid(context['origOid'], text, SnmprecRecord())
exactMatch = False
else:
offset, subtreeFlag, prevOffset = line.split(str2octs(','))
exactMatch = True
text.seek(int(offset))
line, _, _ = getRecord(text) # matched line
if context['nextFlag']:
if exactMatch:
line, _, _ = getRecord(text)
else:
if not exactMatch:
return context['origOid'], tag, context['errorStatus']
if not line:
return context['origOid'], tag, context['errorStatus']
try:
oid, value = SnmprecRecord().evaluate(line)
except error.SnmpsimError:
oid, value = context['origOid'], context['errorStatus']
return oid, tag, value
