class Poller(Task):
def __init__(self, **kwargs):
self.standard_mibs = []
self.mongo_client = pymongo.MongoClient(MONGO_URI)
if kwargs.get("no_mongo"):
self.session = Session()
else:
self.session = CachedLimiterSession(
per_second=120,
cache_name="cache_http",
backend=MongoCache(connection=self.mongo_client,
db_name=MONGO_DB),
expire_after=1800,
match_headers=False,
stale_if_error=True,
allowable_codes=[200],
)
self.profiles = load_profiles()
self.last_modified = time.time()
self.snmpEngine = SnmpEngine()
self.builder = self.snmpEngine.getMibBuilder()
self.mib_view_controller = view.MibViewController(self.builder)
compiler.addMibCompiler(self.builder, sources=[MIB_SOURCES])
# mib_standard_response = self.session.get(f"{MIB_STANDARD}", stream=True)
# if mib_standard_response.status_code == 200:
# for line in mib_standard_response.iter_lines():
# if line:
# mib = line.decode("utf-8")
# logger.info(f"MIB: {mib}")
# try:
# self.builder.loadModules(mib)
# self.standard_mibs.append(mib)
# except Exception as e:
# logger.warning(f"An error occurred during loading MIB module: {mib}: {e}")
# else:
for mib in DEFAULT_STANDARD_MIBS:
self.standard_mibs.append(mib)
self.builder.loadModules(mib)
mib_response = self.session.get(f"{MIB_INDEX}")
self.mib_map = {}
if mib_response.status_code == 200:
with StringIO(mib_response.text) as index_csv:
reader = csv.reader(index_csv)
for each_row in reader:
if len(each_row) == 2:
self.mib_map[each_row[1]] = each_row[0]
logger.debug(f"Loaded {len(self.mib_map.keys())} mib map entries")
else:
logger.error(
f"Unable to load mib map from index http error {self.mib_response.status_code}"
)
def do_work(
self,
ir: InventoryRecord,
walk: bool = False,
profiles: List[str] = None,
walked_first_time=True,
):
retry = False
address = transform_address_to_key(ir.address, ir.port)
if time.time() - self.last_modified > PROFILES_RELOAD_DELAY:
self.profiles = load_profiles()
self.last_modified = time.time()
logger.debug("Profiles reloaded")
varbinds_get, get_mapping, varbinds_bulk, bulk_mapping = self.get_var_binds(
address,
walk=walk,
profiles=profiles,
walked_first_time=walked_first_time)
authData = GetAuth(logger, ir, self.snmpEngine)
contextData = get_context_data()
transport = UdpTransportTarget((ir.address, ir.port),
timeout=UDP_CONNECTION_TIMEOUT)
metrics: Dict[str, Any] = {}
if not varbinds_get and not varbinds_bulk:
logger.info(f"No work to do for {address}")
return False, {}
if varbinds_bulk:
for (
errorIndication,
errorStatus,
errorIndex,
varBindTable,
) in bulkCmd(
self.snmpEngine,
authData,
transport,
contextData,
1,
10,
*varbinds_bulk,
lexicographicMode=False,
ignoreNonIncreasingOid=is_increasing_oids_ignored(
ir.address, ir.port),
):
if not _any_failure_happened(
errorIndication,
errorStatus,
errorIndex,
varBindTable,
ir.address,
walk,
):
tmp_retry, tmp_mibs, _ = self.process_snmp_data(
varBindTable, metrics, address, bulk_mapping)
if tmp_mibs:
self.load_mibs(tmp_mibs)
self.process_snmp_data(varBindTable, metrics, address,
bulk_mapping)
if varbinds_get:
for (
errorIndication,
errorStatus,
errorIndex,
varBindTable,
) in getCmd(self.snmpEngine, authData, transport, contextData,
*varbinds_get):
if not _any_failure_happened(
errorIndication,
errorStatus,
errorIndex,
varBindTable,
ir.address,
walk,
):
self.process_snmp_data(varBindTable, metrics, address,
get_mapping)
for group_key, metric in metrics.items():
if "profiles" in metrics[group_key]:
metrics[group_key]["profiles"] = ",".join(
metrics[group_key]["profiles"])
return retry, metrics
def load_mibs(self, mibs: List[str]) -> None:
logger.info(f"loading mib modules {mibs}")
for mib in mibs:
if mib:
try:
self.builder.loadModules(mib)
except error.MibLoadError as e:
logger.exception(f"Error loading mib for {mib}, {e}")
def is_mib_known(self, id: str, oid: str, target: str) -> tuple[bool, str]:
oid_list = tuple(oid.split("."))
start = 5
for i in range(len(oid_list), start, -1):
oid_to_check = ".".join(oid_list[:i])
if oid_to_check in self.mib_map:
mib = self.mib_map[oid_to_check]
logger.debug(f"found {mib} for {id} based on {oid_to_check}")
return True, mib
logger.warning(f"no mib found {id} based on {oid} from {target}")
return False, ""
def get_var_binds(self,
address,
walk=False,
profiles=[],
walked_first_time=True):
varbinds_bulk = set()
varbinds_get = set()
get_mapping = {}
bulk_mapping = {}
if walk and (not walked_first_time or not profiles):
varbinds_bulk.add(ObjectType(ObjectIdentity("1.3.6")))
else:
needed_mibs = []
required_bulk = {}
# First pass we only look at profiles for a full mib walk
for profile in profiles:
# Its possible a profile is removed on upgrade but schedule doesn't yet know
if profile in self.profiles and "varBinds" in self.profiles[
profile]:
profile_spec = self.profiles[profile]
profile_varbinds = profile_spec["varBinds"]
for vb in profile_varbinds:
if len(vb) == 1:
if vb[0] not in required_bulk:
required_bulk[vb[0]] = None
if not walk:
bulk_mapping[f"{vb[0]}"] = profile
if vb[0] not in needed_mibs:
needed_mibs.append(vb[0])
for profile in profiles:
# Its possible a profile is removed on upgrade but schedule doesn't yet know
if profile in self.profiles and "varBinds" in self.profiles[
profile]:
profile_spec = self.profiles[profile]
profile_varbinds = profile_spec["varBinds"]
for vb in profile_varbinds:
if len(vb) == 2:
if vb[0] not in required_bulk or (
required_bulk[vb[0]]
and vb[1] not in required_bulk[vb[0]]):
if vb[0] not in required_bulk:
required_bulk[vb[0]] = [vb[1]]
else:
required_bulk[vb[0]].append(vb[1])
if not walk:
bulk_mapping[f"{vb[0]}:{vb[1]}"] = profile
for mib, entries in required_bulk.items():
if entries is None:
varbinds_bulk.add(ObjectType(ObjectIdentity(mib)))
else:
for entry in entries:
varbinds_bulk.add(
ObjectType(ObjectIdentity(mib, entry)))
for profile in profiles:
# Its possible a profile is removed on upgrade but schedule doesn't yet know
if profile in self.profiles and "varBinds" in self.profiles[
profile]:
profile_spec = self.profiles[profile]
profile_varbinds = profile_spec["varBinds"]
for vb in profile_varbinds:
if len(vb) == 3:
if vb[0] not in required_bulk or (
required_bulk[vb[0]]
and vb[1] not in required_bulk[vb[0]]):
varbinds_get.add(
ObjectType(
ObjectIdentity(vb[0], vb[1], vb[2])))
if not walk:
get_mapping[
f"{vb[0]}:{vb[1]}:{vb[2]}"] = profile
self.load_mibs(needed_mibs)
logger.debug(f"host={address} varbinds_get={varbinds_get}")
logger.debug(f"host={address} get_mapping={get_mapping}")
logger.debug(f"host={address} varbinds_bulk={varbinds_bulk}")
logger.debug(f"host={address} bulk_mapping={bulk_mapping}")
return varbinds_get, get_mapping, varbinds_bulk, bulk_mapping
def process_snmp_data(self, varBindTable, metrics, target, mapping={}):
i = 0
retry = False
remotemibs = []
for varBind in varBindTable:
i += 1
mib, metric, index = varBind[0].getMibSymbol()
id = varBind[0].prettyPrint()
oid = str(varBind[0].getOid())
if isMIBResolved(id):
group_key = get_group_key(mib, oid, index)
if group_key not in metrics:
metrics[group_key] = {
"metrics": {},
"fields": {},
}
if mapping:
metrics[group_key]["profiles"] = []
try:
snmp_val = varBind[1]
snmp_type = type(snmp_val).__name__
metric_type = map_metric_type(snmp_type, snmp_val)
metric_value = valueAsBest(snmp_val.prettyPrint())
index_number = extract_index_number(index)
metric_value = fill_empty_value(index_number, metric_value,
target)
profile = None
if mapping:
profile = mapping.get(
f"{mib}:{metric}:{index_number}",
mapping.get(f"{mib}:{metric}", mapping.get(mib)),
)
if metric_value == "No more variables left in this MIB View":
continue
if metric_type in MTYPES and (isinstance(
metric_value, float)):
metrics[group_key]["metrics"][f"{mib}.{metric}"] = {
"time": time.time(),
"type": metric_type,
"value": metric_value,
"oid": oid,
}
if profile and profile not in metrics[group_key][
"profiles"]:
metrics[group_key]["profiles"].append(profile)
else:
metrics[group_key]["fields"][f"{mib}.{metric}"] = {
"time": time.time(),
"type": metric_type,
"value": metric_value,
"oid": oid,
}
except:
logger.exception(
f"Exception processing data from {target} {varBind}")
else:
found, mib = self.is_mib_known(id, oid, target)
if mib and mib not in remotemibs:
remotemibs.append(mib)
if found:
retry = True
break
return retry, remotemibs, metrics
def get_engine():
mibBuilder = builder.MibBuilder()
engine=SnmpEngine()
builder_engine = engine.getMibBuilder()
builder_engine.addMibSources(builder.DirMibSource('Scripts/python/snmp/mibs'))
return engine
