def variate(oid, tag, value, **context):
if 'settings' not in recordContext:
recordContext['settings'] = dict(
[split(x, '=') for x in split(value, ',')])
if 'dir' not in recordContext['settings']:
log.msg('multiplex: snapshot directory not specified')
return context['origOid'], tag, context['errorStatus']
recordContext['settings']['dir'] = recordContext['settings'][
'dir'].replace('/', os.path.sep)
if recordContext['settings']['dir'][0] != os.path.sep:
for x in confdir.data:
d = os.path.join(x, recordContext['settings']['dir'])
if os.path.exists(d):
break
else:
log.msg('multiplex: directory %s not found' %
recordContext['settings']['dir'])
return context['origOid'], tag, context['errorStatus']
else:
d = recordContext['settings']['dir']
recordContext['dirmap'] = dict([
(int(os.path.basename(x).split(os.path.extsep)[0]),
os.path.join(d, x)) for x in os.listdir(d) if x[-7:] == 'snmprec'
])
recordContext['keys'] = list(recordContext['dirmap'].keys())
recordContext['bounds'] = (min(recordContext['keys']),
max(recordContext['keys']))
if 'period' in recordContext['settings']:
recordContext['settings']['period'] = float(
recordContext['settings']['period'])
else:
recordContext['settings']['period'] = 60.0
if 'wrap' in recordContext['settings']:
recordContext['settings']['wrap'] = bool(
recordContext['settings']['wrap'])
else:
recordContext['settings']['wrap'] = False
if 'control' in recordContext['settings']:
recordContext['settings']['control'] = rfc1902.ObjectName(
recordContext['settings']['control'])
log.msg(
'multiplex: using control OID %s for subtree %s, time-based multiplexing disabled'
% (recordContext['settings']['control'], oid))
recordContext['ready'] = True
if 'ready' not in recordContext:
return context['origOid'], tag, context['errorStatus']
if oid not in moduleContext:
moduleContext[oid] = {}
if context['setFlag']:
if 'control' in recordContext['settings'] and \
recordContext['settings']['control'] == context['origOid']:
fileno = int(context['origValue'])
if fileno >= len(recordContext['keys']):
log.msg('multiplex: .snmprec file number %s over limit of %s' %
(fileno, len(recordContext['keys'])))
return context['origOid'], tag, context['errorStatus']
moduleContext[oid]['fileno'] = fileno
log.msg('multiplex: switched to file #%s (%s)' %
(recordContext['keys'][fileno],
recordContext['dirmap'][recordContext['keys'][fileno]]))
return context['origOid'], tag, context['origValue']
else:
return context['origOid'], tag, context['errorStatus']
if 'control' in recordContext['settings']:
if 'fileno' not in moduleContext[oid]:
moduleContext[oid]['fileno'] = 0
if (not context['nextFlag'] and recordContext['settings']['control']
== context['origOid']):
return context['origOid'], tag, rfc1902.Integer32(
moduleContext[oid]['fileno'])
else:
timeslot = (time.time() - moduleContext['booted']) % (
recordContext['settings']['period'] * len(recordContext['dirmap']))
fileslot = int(
timeslot /
recordContext['settings']['period']) + recordContext['bounds'][0]
fileno = bisect.bisect(recordContext['keys'], fileslot) - 1
if ('fileno' not in moduleContext[oid]
or moduleContext[oid]['fileno'] < fileno
or recordContext['settings']['wrap']):
moduleContext[oid]['fileno'] = fileno
datafile = recordContext['dirmap'][recordContext['keys'][moduleContext[oid]
['fileno']]]
if ('datafile' not in moduleContext[oid]
or moduleContext[oid]['datafile'] != datafile):
if 'datafileobj' in moduleContext[oid]:
moduleContext[oid]['datafileobj'].close()
moduleContext[oid]['datafileobj'] = RecordIndex(
datafile, SnmprecRecord()).create()
moduleContext[oid]['datafile'] = datafile
log.msg('multiplex: switching to data file %s for %s' %
(datafile, context['origOid']))
text, db = moduleContext[oid]['datafileobj'].getHandles()
textOid = str(
rfc1902.OctetString('.'.join(['%s' % x for x in context['origOid']])))
try:
line = moduleContext[oid]['datafileobj'].lookup(textOid)
except KeyError:
offset = searchRecordByOid(context['origOid'], text, SnmprecRecord())
exactMatch = False
else:
offset, subtreeFlag, prevOffset = line.split(str2octs(','))
exactMatch = True
text.seek(int(offset))
line, _, _ = getRecord(text) # matched line
if context['nextFlag']:
if exactMatch:
line, _, _ = getRecord(text)
else:
if not exactMatch:
return context['origOid'], tag, context['errorStatus']
if not line:
return context['origOid'], tag, context['errorStatus']
try:
oid, value = SnmprecRecord().evaluate(line)
except error.SnmpsimError:
oid, value = context['origOid'], context['errorStatus']
return oid, tag, value
def __init__(self, textFile, textParser, variationModules):
self._record_index = RecordIndex(textFile, textParser)
self._text_parser = textParser
self._text_file = textFile
self._variation_modules = variationModules
def variate(oid, tag, value, **context):
if 'settings' not in recordContext:
recordContext['settings'] = dict(
[split(x, '=') for x in split(value, ',')])
if 'dir' not in recordContext['settings']:
log.info('multiplex: snapshot directory not specified')
return context['origOid'], tag, context['errorStatus']
recordContext['settings']['dir'] = recordContext['settings'][
'dir'].replace('/', os.path.sep)
if recordContext['settings']['dir'][0] != os.path.sep:
for x in confdir.data:
d = os.path.join(x, recordContext['settings']['dir'])
if os.path.exists(d):
break
else:
log.info('multiplex: directory %s not '
'found' % recordContext['settings']['dir'])
return context['origOid'], tag, context['errorStatus']
else:
d = recordContext['settings']['dir']
recordContext['dirmap'] = {}
recordContext['parsermap'] = {}
for fl in os.listdir(d):
for ext in RECORD_SET:
if not fl.endswith(ext):
continue
ident = int(os.path.basename(fl)[:-len(ext) - 1])
datafile = os.path.join(d, fl)
recordContext['dirmap'][ident] = datafile
recordContext['parsermap'][datafile] = RECORD_SET[ext]
recordContext['keys'] = list(recordContext['dirmap'])
recordContext['bounds'] = (min(recordContext['keys']),
max(recordContext['keys']))
if 'period' in recordContext['settings']:
recordContext['settings']['period'] = float(
recordContext['settings']['period'])
else:
recordContext['settings']['period'] = 60.0
if 'wrap' in recordContext['settings']:
recordContext['settings']['wrap'] = bool(
recordContext['settings']['wrap'])
else:
recordContext['settings']['wrap'] = False
if 'control' in recordContext['settings']:
recordContext['settings']['control'] = rfc1902.ObjectName(
recordContext['settings']['control'])
log.info('multiplex: using control OID %s for subtree %s, '
'time-based multiplexing '
'disabled' % (recordContext['settings']['control'], oid))
recordContext['ready'] = True
if 'ready' not in recordContext:
return context['origOid'], tag, context['errorStatus']
if oid not in moduleContext:
moduleContext[oid] = {}
if context['setFlag']:
if 'control' in (recordContext['settings']
and recordContext['settings']['control']
== context['origOid']):
fileno = int(context['origValue'])
if fileno >= len(recordContext['keys']):
log.info('multiplex: .snmprec file number %s over limit of'
' %s' % (fileno, len(recordContext['keys'])))
return context['origOid'], tag, context['errorStatus']
moduleContext[oid]['fileno'] = fileno
log.info('multiplex: switched to file #%s '
'(%s)' %
(recordContext['keys'][fileno],
recordContext['dirmap'][recordContext['keys'][fileno]]))
return context['origOid'], tag, context['origValue']
else:
return context['origOid'], tag, context['errorStatus']
if 'control' in recordContext['settings']:
if 'fileno' not in moduleContext[oid]:
moduleContext[oid]['fileno'] = 0
if (not context['nextFlag'] and recordContext['settings']['control']
== context['origOid']):
val = rfc1902.Integer32(moduleContext[oid]['fileno'])
return context['origOid'], tag, val
else:
period = recordContext['settings']['period']
uptime = time.time() - moduleContext['booted']
timeslot = uptime % (period * len(recordContext['dirmap']))
fileslot = int(timeslot / period) + recordContext['bounds'][0]
fileno = bisect.bisect(recordContext['keys'], fileslot) - 1
if ('fileno' not in moduleContext[oid]
or moduleContext[oid]['fileno'] < fileno
or recordContext['settings']['wrap']):
moduleContext[oid]['fileno'] = fileno
datafile = recordContext['dirmap'][recordContext['keys'][moduleContext[oid]
['fileno']]]
parser = recordContext['parsermap'][datafile]
if ('datafile' not in moduleContext[oid]
or moduleContext[oid]['datafile'] != datafile):
if 'datafileobj' in moduleContext[oid]:
moduleContext[oid]['datafileobj'].close()
recordIndex = RecordIndex(datafile, parser).create()
moduleContext[oid]['datafileobj'] = recordIndex
moduleContext[oid]['datafile'] = datafile
log.info('multiplex: switching to data file %s for '
'%s' % (datafile, context['origOid']))
text, db = moduleContext[oid]['datafileobj'].get_handles()
textOid = str(
rfc1902.OctetString('.'.join(['%s' % x for x in context['origOid']])))
try:
line = moduleContext[oid]['datafileobj'].lookup(textOid)
except KeyError:
offset = search_record_by_oid(context['origOid'], text, parser)
exactMatch = False
else:
offset, subtreeFlag, prevOffset = line.split(str2octs(','))
exactMatch = True
text.seek(int(offset))
line, _, _ = get_record(text) # matched line
if context['nextFlag']:
if exactMatch:
line, _, _ = get_record(text)
else:
if not exactMatch:
return context['origOid'], tag, context['errorStatus']
if not line:
return context['origOid'], tag, context['errorStatus']
try:
oid, value = parser.evaluate(line)
except error.SnmpsimError:
oid, value = context['origOid'], context['errorStatus']
return oid, tag, value
class DataFile(AbstractLayout):
layout = 'text'
opened_queue = []
max_queue_entries = 31 # max number of open text and index files
def __init__(self, textFile, textParser, variationModules):
self._record_index = RecordIndex(textFile, textParser)
self._text_parser = textParser
self._text_file = textFile
self._variation_modules = variationModules
def index_text(self, forceIndexBuild=False, validateData=False):
self._record_index.create(forceIndexBuild, validateData)
return self
def close(self):
self._record_index.close()
def get_handles(self):
if not self._record_index.is_open():
if len(DataFile.opened_queue) > self.max_queue_entries:
log.info('Closing %s' % self)
DataFile.opened_queue[0].close()
del DataFile.opened_queue[0]
DataFile.opened_queue.append(self)
log.info('Opening %s' % self)
return self._record_index.get_handles()
def process_var_binds(self, var_binds, **context):
rsp_var_binds = []
if context.get('nextFlag'):
error_status = exval.endOfMib
else:
error_status = exval.noSuchInstance
try:
text, db = self.get_handles()
except SnmpsimError as exc:
log.error('Problem with data file or its index: %s' % exc)
ReportingManager.update_metrics(data_file=self._text_file,
datafile_failure_count=1,
transport_call_count=1,
**context)
return [(vb[0], error_status) for vb in var_binds]
vars_remaining = vars_total = len(var_binds)
err_total = 0
log.info(
'Request var-binds: %s, flags: %s, '
'%s' % (', '.join(
['%s=<%s>' % (vb[0], vb[1].prettyPrint())
for vb in var_binds]), context.get('nextFlag') and 'NEXT'
or 'EXACT', context.get('setFlag') and 'SET' or 'GET'))
for oid, val in var_binds:
text_oid = str(univ.OctetString('.'.join(['%s' % x for x in oid])))
try:
line = self._record_index.lookup(
str(univ.OctetString('.'.join(['%s' % x for x in oid]))))
except KeyError:
offset = search_record_by_oid(oid, text, self._text_parser)
subtree_flag = exact_match = False
else:
offset, subtree_flag, prev_offset = line.split(
str2octs(','), 2)
subtree_flag, exact_match = int(subtree_flag), True
offset = int(offset)
text.seek(offset)
vars_remaining -= 1
line, _, _ = get_record(text) # matched line
while True:
if exact_match:
if context.get('nextFlag') and not subtree_flag:
_next_line, _, _ = get_record(text) # next line
if _next_line:
_next_oid, _ = self._text_parser.evaluate(
_next_line, oidOnly=True)
try:
_, subtree_flag, _ = self._record_index.lookup(
str(_next_oid)).split(str2octs(','), 2)
except KeyError:
log.error('data error for %s at %s, index '
'broken?' % (self, _next_oid))
line = '' # fatal error
else:
subtree_flag = int(subtree_flag)
line = _next_line
else:
line = _next_line
else: # search function above always rounds up to the next OID
if line:
_oid, _ = self._text_parser.evaluate(line,
oidOnly=True)
else: # eom
_oid = 'last'
try:
_, _, _prev_offset = self._record_index.lookup(
str(_oid)).split(str2octs(','), 2)
except KeyError:
log.error('data error for %s at %s, index '
'broken?' % (self, _oid))
line = '' # fatal error
else:
_prev_offset = int(_prev_offset)
# previous line serves a subtree?
if _prev_offset >= 0:
text.seek(_prev_offset)
_prev_line, _, _ = get_record(text)
_prev_oid, _ = self._text_parser.evaluate(
_prev_line, oidOnly=True)
if _prev_oid.isPrefixOf(oid):
# use previous line to the matched one
line = _prev_line
subtree_flag = True
if not line:
_oid = oid
_val = error_status
break
call_context = context.copy()
call_context.update((),
origOid=oid,
origValue=val,
dataFile=self._text_file,
subtreeFlag=subtree_flag,
exactMatch=exact_match,
errorStatus=error_status,
varsTotal=vars_total,
varsRemaining=vars_remaining,
variationModules=self._variation_modules)
try:
_oid, _val = self._text_parser.evaluate(
line, **call_context)
if _val is exval.endOfMib:
exact_match = True
subtree_flag = False
continue
except NoDataNotification:
raise
except MibOperationError:
raise
except Exception as exc:
_oid = oid
_val = error_status
err_total += 1
log.error('data error at %s for %s: %s' %
(self, text_oid, exc))
break
rsp_var_binds.append((_oid, _val))
log.info('Response var-binds: %s' % (', '.join(
['%s=<%s>' % (vb[0], vb[1].prettyPrint())
for vb in rsp_var_binds])))
ReportingManager.update_metrics(data_file=self._text_file,
varbind_count=vars_total,
datafile_call_count=1,
datafile_failure_count=err_total,
transport_call_count=1,
**context)
return rsp_var_binds
def __str__(self):
return '%s controller' % self._text_file