def __init__(self, mib_list, src_dir_list, http_sources=None):
if "BFT_DEBUG" in os.environ:
self.dbg = os.environ.get('BFT_DEBUG')
else:
self.dbg = ""
if "yy" in self.dbg:
# VERY verbose, but essential for spotting
# possible ASN.1 errors
from pysmi import debug
debug.setLogger(debug.Debug('reader', 'compiler'))
# Initialize compiler infrastructure
mibCompiler = MibCompiler(SmiStarParser(), JsonCodeGen(),
CallbackWriter(self.callback_func))
# search for source MIBs here
mibCompiler.addSources(*[FileReader(x) for x in src_dir_list])
if http_sources:
# search for source MIBs at Web sites
mibCompiler.addSources(*[HttpReader(*x) for x in http_sources])
# never recompile MIBs with MACROs
mibCompiler.addSearchers(StubSearcher(*JsonCodeGen.baseMibs))
# run recursive MIB compilation
mibCompiler.compile(*mib_list)
def _compile_mib_to_json(mib, source_mib_directories, destination_directory,
source, compiled_mibs_path):
from pysmi.borrower import AnyFileBorrower
from pysmi.codegen import JsonCodeGen
from pysmi.compiler import MibCompiler
from pysmi.parser import SmiV1CompatParser
from pysmi.reader import getReadersFromUrls
from pysmi.searcher import AnyFileSearcher, StubSearcher
from pysmi.writer import FileWriter
mib_stubs = JsonCodeGen.baseMibs
compile_documentation = True
# Compiler infrastructure
code_generator = JsonCodeGen()
file_writer = FileWriter(destination_directory).setOptions(suffix='.json')
mib_compiler = MibCompiler(SmiV1CompatParser(tempdir=''), code_generator,
file_writer)
# use source_mib_directories as mibs source
sources = [source]
sources.extend(source_mib_directories)
mib_compiler.addSources(
*getReadersFromUrls(*sources, **dict(fuzzyMatching=True)))
searchers = [
AnyFileSearcher(destination_directory).setOptions(exts=['.json']),
StubSearcher(*mib_stubs)
]
mib_compiler.addSearchers(*searchers)
# borrowers, aka compiled mibs source
borrowers = [
AnyFileBorrower(borrower_reader,
genTexts=True).setOptions(exts=['.json'])
for borrower_reader in getReadersFromUrls(
*[compiled_mibs_path], **dict(lowcaseMatching=False))
]
mib_compiler.addBorrowers(borrowers)
processed = mib_compiler.compile(
mib,
**dict(
noDeps=False,
rebuild=False,
dryRun=False,
dstTemplate=None,
genTexts=compile_documentation,
textFilter=False and (lambda symbol, text: text) or None,
writeMibs=True,
ignoreErrors=False,
))
return processed
def _compile_mib_to_json(mib, source_mib_directories, destination_directory):
from pysmi.codegen import JsonCodeGen
from pysmi.compiler import MibCompiler
from pysmi.parser import SmiV1CompatParser
from pysmi.reader import FileReader, HttpReader
from pysmi.searcher import AnyFileSearcher, StubSearcher
from pysmi.writer import FileWriter
mib_stubs = JsonCodeGen.baseMibs
compile_documentation = True
# Compiler infrastructure
searchers = [
AnyFileSearcher(destination_directory).setOptions(exts=['.json']),
StubSearcher(*mib_stubs)
]
code_generator = JsonCodeGen()
file_writer = FileWriter(destination_directory).setOptions(suffix='.json')
mib_compiler = MibCompiler(SmiV1CompatParser(tempdir=''), code_generator,
file_writer)
# use source_mib_directories as mibs source
for source_directory in source_mib_directories:
mib_compiler.addSources(FileReader(source_directory))
# use snmp mibs repo as mibs source
reader = HttpReader('raw.githubusercontent.com', 80,
'/projx/snmp-mibs/master/@mib@')
mib_compiler.addSources(reader)
mib_compiler.addSearchers(*searchers)
processed = mib_compiler.compile(
mib,
**dict(
noDeps=False,
rebuild=False,
dryRun=False,
dstTemplate=None,
genTexts=compile_documentation,
textFilter=False and (lambda symbol, text: text) or None,
writeMibs=True,
ignoreErrors=False,
))
return processed
def __init__(self, mib_list, src_dir_list, http_sources=None):
if "BFT_DEBUG" in os.environ:
self.dbg = os.environ.get('BFT_DEBUG')
else:
self.dbg = ""
if "yy" in self.dbg:
# VERY verbose, but essential for spotting
# possible ASN.1 errors
from pysmi import debug
debug.setLogger(debug.Debug('reader', 'compiler'))
# Initialize compiler infrastructure
mibCompiler = MibCompiler(SmiStarParser(), JsonCodeGen(),
CallbackWriter(self.callback_func))
# search for source MIBs here
mibCompiler.addSources(*[FileReader(x) for x in src_dir_list])
if http_sources:
# search for source MIBs at Web sites
mibCompiler.addSources(*[HttpReader(*x) for x in http_sources])
# never recompile MIBs with MACROs
mibCompiler.addSearchers(StubSearcher(*JsonCodeGen.baseMibs))
# run recursive MIB compilation
mib_dict = mibCompiler.compile(*mib_list)
err = False
if mib_dict is None or mib_dict == {}:
print(
"ERROR: failed on mib compilation (mibCompiler.compile returned an empty dictionary)"
)
err = True
for key, value in mib_dict.iteritems():
if value == 'unprocessed':
print("ERROR: failed on mib compilation: " + key + ": " +
value)
err = True
if err:
raise Exception("SnmpMibs failed to initialize.")
elif 'BFT_DEBUG' in os.environ:
print('# %d MIB modules compiled' % len(mib_dict))
def transform_mibs_to_json(self,
specific_mibs: object = None,
exclude_mibs: object = []) -> object:
mib_names = []
if specific_mibs is None:
for filename in os.listdir(self.local_mib_dir):
try:
filename.rindex('.my', -3)
except:
continue
mib_names.append(filename)
else:
mib_names = specific_mibs.copy()
logging.info('Compiling MIBs to JSON.')
try:
mib_compiler = MibCompiler(
SmiV1CompatParser(), JsonCodeGen(),
FileWriter(self.local_json_dir).setOptions(suffix='.json'))
mib_compiler.addSources(
FileReader(self.local_mib_dir, recursive=True))
mib_stubs = JsonCodeGen.baseMibs
searchers = [
AnyFileSearcher(
self.local_json_dir).setOptions(exts=['.json']),
StubSearcher(*mib_stubs)
]
mib_compiler.addSearchers(*searchers)
if not os.path.isdir(self.local_json_dir):
os.makedirs(self.local_json_dir)
processed = mib_compiler.compile(
*mib_names,
**dict(noDeps=False,
rebuild=True,
genTexts=True,
writeMibs=True,
ignoreErrors=True))
mib_compiler.buildIndex(processed, ignoreErrors=True)
except error.PySmiError:
logging.error('ERROR: %s', str(sys.exc_info()[1]))
sys.exit(1)
logging.info('Finished compiling MIBs to JSON.')
dstDirectory = inputMibs.pop()
if os.path.exists(dstDirectory) and not os.path.isdir(dstDirectory):
sys.stderr.write('ERROR: given destination is not a directory\r\n%s\r\n' %
helpMessage)
sys.exit(EX_USAGE)
try:
os.makedirs(dstDirectory, mode=0o755)
except OSError:
pass
# Compiler infrastructure
codeGenerator = JsonCodeGen()
mibParser = SmiV1CompatParser(tempdir=cacheDirectory)
fileWriter = CallbackWriter(lambda *x: None)
def getMibRevision(mibDir, mibFile):
mibCompiler = MibCompiler(mibParser, codeGenerator, fileWriter)
mibCompiler.addSources(
FileReader(mibDir, recursive=False, ignoreErrors=ignoreErrorsFlag),
*getReadersFromUrls(*mibSources))
try:
from pysmi.reader import HttpReader
from pysmi.searcher import StubSearcher
from pysmi.writer import CallbackWriter
from pysmi.parser import SmiStarParser
from pysmi.codegen import JsonCodeGen
from pysmi.compiler import MibCompiler
inputMibs = ['IF-MIB', 'IP-MIB']
httpSources = [('mibs.snmplabs.com', 80, '/asn1/@mib@')]
# store compiled MIBs by calling this function
def store_mibs(mibName, jsonDoc, cbCtx):
print('# MIB module %s' % mibName)
print(jsonDoc)
mibCompiler = MibCompiler(SmiStarParser(), JsonCodeGen(),
CallbackWriter(store_mibs))
# pull ASN.1 MIBs over HTTP
mibCompiler.addSources(*[HttpReader(*x) for x in httpSources])
# never recompile MIBs with ASN.1 MACROs
mibCompiler.addSearchers(StubSearcher(*JsonCodeGen.baseMibs))
status = mibCompiler.compile(*inputMibs)
print(status)
def writer(mib_name, json_doc, context):
data = json.loads(json_doc)
for k,v in data.items():
DATA[v.get('oid')] = v.get('name')
DATA[v.get('name')] = v.get('oid')
for inputMib in inputMibs:
print(inputMib)
try:
mibCompiler = MibCompiler(
SmiStarParser(), JsonCodeGen(), CallbackWriter(writer)
)
# search for source MIBs here
mibCompiler.addSources(*[FileReader(x) for x in srcDirectories])
# search for source MIBs at Web sites
#mibCompiler.addSources(*[HttpReader(*x) for x in httpSources])
# never recompile MIBs with MACROs
#mibCompiler.addSearchers(StubSearcher(*JsonCodeGen.baseMibs))
# run recursive MIB compilation
results = mibCompiler.compile(inputMib)
except Exception as e:
def generate_traps_db(mib_sources, output_dir, output_file, output_format,
no_descr, debug, mib_files):
"""Generate yaml or json formatted documents containing various information about traps. These files can be used by
the Datadog Agent to enrich trap data.
This command is intended for "Network Devices Monitoring" users who need to enrich traps that are not automatically
supported by Datadog.
The expected workflow is as such:\n
1- Identify a type of device that is sending traps that Datadog does not already recognize.\n
2- Fetch all the MIBs that Datadog does not support.\n
3- Run `ddev meta snmp generate-traps-db -o ./output_dir/ /path/to/my/mib1 /path/to/my/mib2`\n
You'll need to install pysmi manually beforehand.
"""
from pysmi.codegen import JsonCodeGen
from pysmi.compiler import MibCompiler
from pysmi.parser import SmiV1CompatParser
from pysmi.reader import getReadersFromUrls
from pysmi.searcher import AnyFileSearcher
from pysmi.writer import FileWriter
if debug:
set_debug()
from pysmi import debug
debug.setLogger(debug.Debug('all'))
# Defaulting to github.com/DataDog/mibs.snmplabs.com/
mib_sources = [mib_sources] if mib_sources else [MIB_SOURCE_URL]
if output_file:
allowed_extensions = ALLOWED_EXTENSIONS_BY_FORMAT[output_format]
if not any(output_file.endswith(x) for x in allowed_extensions):
abort("Output file {} does not end with an allowed extension '{}'".
format(output_file, ", ".join(allowed_extensions)))
if output_dir and output_file:
abort(
"Do not set both --output-dir and --output-file at the same time.")
elif not output_file and not output_dir:
abort("Need to set one of --output-dir or --output-file")
with TempDir('ddev_mibs') as compiled_mibs_sources:
compiled_mibs_sources = os.path.abspath(compiled_mibs_sources)
echo_info("Writing intermediate compiled MIBs to {}".format(
compiled_mibs_sources))
mibs_sources_dir = os.path.join(compiled_mibs_sources, 'mibs_sources')
if not os.path.isdir(mibs_sources_dir):
os.mkdir(mibs_sources_dir)
mib_sources = (sorted(
set([
os.path.abspath(os.path.dirname(x))
for x in mib_files if os.path.sep in x
])) + mib_sources)
mib_files = [os.path.basename(x) for x in mib_files]
searchers = [
AnyFileSearcher(compiled_mibs_sources).setOptions(exts=['.json'])
]
code_generator = JsonCodeGen()
file_writer = FileWriter(compiled_mibs_sources).setOptions(
suffix='.json')
mib_compiler = MibCompiler(SmiV1CompatParser(tempdir=''),
code_generator, file_writer)
mib_compiler.addSources(
*getReadersFromUrls(*mib_sources, **dict(fuzzyMatching=True)))
mib_compiler.addSearchers(*searchers)
compiled_mibs, compiled_dependencies_mibs = compile_and_report_status(
mib_files, mib_compiler)
# Move all the parent MIBs that had to be compiled but were not requested in the command to a subfolder.
for mib_file_name in compiled_dependencies_mibs:
os.replace(
os.path.join(compiled_mibs_sources, mib_file_name + '.json'),
os.path.join(mibs_sources_dir, mib_file_name + '.json'),
)
# Only generate trap_db with `mib_files` unless explicitly asked. Used to ignore other files that may be
# present "compiled_mibs_sources"
compiled_mibs = [
os.path.join(compiled_mibs_sources, x + '.json')
for x in compiled_mibs
]
# Generate the trap database based on the compiled MIBs.
trap_db_per_mib = generate_trap_db(compiled_mibs, mibs_sources_dir,
no_descr)
use_json = output_format == "json"
if output_file:
# Compact representation, only one file
write_compact_trap_db(trap_db_per_mib,
output_file,
use_json=use_json)
echo_success("Wrote trap data to {}".format(
os.path.abspath(output_file)))
else:
# Expanded representation, one file per MIB.
write_trap_db_per_mib(trap_db_per_mib,
output_dir,
use_json=use_json)
echo_success("Wrote trap data to {}".format(
os.path.abspath(output_dir)))
