class Poller(Task): def __init__(self, **kwargs): self.standard_mibs = [] self.mongo_client = pymongo.MongoClient(MONGO_URI) if kwargs.get("no_mongo"): self.session = Session() else: self.session = CachedLimiterSession( per_second=120, cache_name="cache_http", backend=MongoCache(connection=self.mongo_client, db_name=MONGO_DB), expire_after=1800, match_headers=False, stale_if_error=True, allowable_codes=[200], ) self.profiles = load_profiles() self.last_modified = time.time() self.snmpEngine = SnmpEngine() self.builder = self.snmpEngine.getMibBuilder() self.mib_view_controller = view.MibViewController(self.builder) compiler.addMibCompiler(self.builder, sources=[MIB_SOURCES]) # mib_standard_response = self.session.get(f"{MIB_STANDARD}", stream=True) # if mib_standard_response.status_code == 200: # for line in mib_standard_response.iter_lines(): # if line: # mib = line.decode("utf-8") # logger.info(f"MIB: {mib}") # try: # self.builder.loadModules(mib) # self.standard_mibs.append(mib) # except Exception as e: # logger.warning(f"An error occurred during loading MIB module: {mib}: {e}") # else: for mib in DEFAULT_STANDARD_MIBS: self.standard_mibs.append(mib) self.builder.loadModules(mib) mib_response = self.session.get(f"{MIB_INDEX}") self.mib_map = {} if mib_response.status_code == 200: with StringIO(mib_response.text) as index_csv: reader = csv.reader(index_csv) for each_row in reader: if len(each_row) == 2: self.mib_map[each_row[1]] = each_row[0] logger.debug(f"Loaded {len(self.mib_map.keys())} mib map entries") else: logger.error( f"Unable to load mib map from index http error {self.mib_response.status_code}" ) def do_work( self, ir: InventoryRecord, walk: bool = False, profiles: List[str] = None, walked_first_time=True, ): retry = False address = transform_address_to_key(ir.address, ir.port) if time.time() - self.last_modified > PROFILES_RELOAD_DELAY: self.profiles = load_profiles() self.last_modified = time.time() logger.debug("Profiles reloaded") varbinds_get, get_mapping, varbinds_bulk, bulk_mapping = self.get_var_binds( address, walk=walk, profiles=profiles, walked_first_time=walked_first_time) authData = GetAuth(logger, ir, self.snmpEngine) contextData = get_context_data() transport = UdpTransportTarget((ir.address, ir.port), timeout=UDP_CONNECTION_TIMEOUT) metrics: Dict[str, Any] = {} if not varbinds_get and not varbinds_bulk: logger.info(f"No work to do for {address}") return False, {} if varbinds_bulk: for ( errorIndication, errorStatus, errorIndex, varBindTable, ) in bulkCmd( self.snmpEngine, authData, transport, contextData, 1, 10, *varbinds_bulk, lexicographicMode=False, ignoreNonIncreasingOid=is_increasing_oids_ignored( ir.address, ir.port), ): if not _any_failure_happened( errorIndication, errorStatus, errorIndex, varBindTable, ir.address, walk, ): tmp_retry, tmp_mibs, _ = self.process_snmp_data( varBindTable, metrics, address, bulk_mapping) if tmp_mibs: self.load_mibs(tmp_mibs) self.process_snmp_data(varBindTable, metrics, address, bulk_mapping) if varbinds_get: for ( errorIndication, errorStatus, errorIndex, varBindTable, ) in getCmd(self.snmpEngine, authData, transport, contextData, *varbinds_get): if not _any_failure_happened( errorIndication, errorStatus, errorIndex, varBindTable, ir.address, walk, ): self.process_snmp_data(varBindTable, metrics, address, get_mapping) for group_key, metric in metrics.items(): if "profiles" in metrics[group_key]: metrics[group_key]["profiles"] = ",".join( metrics[group_key]["profiles"]) return retry, metrics def load_mibs(self, mibs: List[str]) -> None: logger.info(f"loading mib modules {mibs}") for mib in mibs: if mib: try: self.builder.loadModules(mib) except error.MibLoadError as e: logger.exception(f"Error loading mib for {mib}, {e}") def is_mib_known(self, id: str, oid: str, target: str) -> tuple[bool, str]: oid_list = tuple(oid.split(".")) start = 5 for i in range(len(oid_list), start, -1): oid_to_check = ".".join(oid_list[:i]) if oid_to_check in self.mib_map: mib = self.mib_map[oid_to_check] logger.debug(f"found {mib} for {id} based on {oid_to_check}") return True, mib logger.warning(f"no mib found {id} based on {oid} from {target}") return False, "" def get_var_binds(self, address, walk=False, profiles=[], walked_first_time=True): varbinds_bulk = set() varbinds_get = set() get_mapping = {} bulk_mapping = {} if walk and (not walked_first_time or not profiles): varbinds_bulk.add(ObjectType(ObjectIdentity("1.3.6"))) else: needed_mibs = [] required_bulk = {} # First pass we only look at profiles for a full mib walk for profile in profiles: # Its possible a profile is removed on upgrade but schedule doesn't yet know if profile in self.profiles and "varBinds" in self.profiles[ profile]: profile_spec = self.profiles[profile] profile_varbinds = profile_spec["varBinds"] for vb in profile_varbinds: if len(vb) == 1: if vb[0] not in required_bulk: required_bulk[vb[0]] = None if not walk: bulk_mapping[f"{vb[0]}"] = profile if vb[0] not in needed_mibs: needed_mibs.append(vb[0]) for profile in profiles: # Its possible a profile is removed on upgrade but schedule doesn't yet know if profile in self.profiles and "varBinds" in self.profiles[ profile]: profile_spec = self.profiles[profile] profile_varbinds = profile_spec["varBinds"] for vb in profile_varbinds: if len(vb) == 2: if vb[0] not in required_bulk or ( required_bulk[vb[0]] and vb[1] not in required_bulk[vb[0]]): if vb[0] not in required_bulk: required_bulk[vb[0]] = [vb[1]] else: required_bulk[vb[0]].append(vb[1]) if not walk: bulk_mapping[f"{vb[0]}:{vb[1]}"] = profile for mib, entries in required_bulk.items(): if entries is None: varbinds_bulk.add(ObjectType(ObjectIdentity(mib))) else: for entry in entries: varbinds_bulk.add( ObjectType(ObjectIdentity(mib, entry))) for profile in profiles: # Its possible a profile is removed on upgrade but schedule doesn't yet know if profile in self.profiles and "varBinds" in self.profiles[ profile]: profile_spec = self.profiles[profile] profile_varbinds = profile_spec["varBinds"] for vb in profile_varbinds: if len(vb) == 3: if vb[0] not in required_bulk or ( required_bulk[vb[0]] and vb[1] not in required_bulk[vb[0]]): varbinds_get.add( ObjectType( ObjectIdentity(vb[0], vb[1], vb[2]))) if not walk: get_mapping[ f"{vb[0]}:{vb[1]}:{vb[2]}"] = profile self.load_mibs(needed_mibs) logger.debug(f"host={address} varbinds_get={varbinds_get}") logger.debug(f"host={address} get_mapping={get_mapping}") logger.debug(f"host={address} varbinds_bulk={varbinds_bulk}") logger.debug(f"host={address} bulk_mapping={bulk_mapping}") return varbinds_get, get_mapping, varbinds_bulk, bulk_mapping def process_snmp_data(self, varBindTable, metrics, target, mapping={}): i = 0 retry = False remotemibs = [] for varBind in varBindTable: i += 1 mib, metric, index = varBind[0].getMibSymbol() id = varBind[0].prettyPrint() oid = str(varBind[0].getOid()) if isMIBResolved(id): group_key = get_group_key(mib, oid, index) if group_key not in metrics: metrics[group_key] = { "metrics": {}, "fields": {}, } if mapping: metrics[group_key]["profiles"] = [] try: snmp_val = varBind[1] snmp_type = type(snmp_val).__name__ metric_type = map_metric_type(snmp_type, snmp_val) metric_value = valueAsBest(snmp_val.prettyPrint()) index_number = extract_index_number(index) metric_value = fill_empty_value(index_number, metric_value, target) profile = None if mapping: profile = mapping.get( f"{mib}:{metric}:{index_number}", mapping.get(f"{mib}:{metric}", mapping.get(mib)), ) if metric_value == "No more variables left in this MIB View": continue if metric_type in MTYPES and (isinstance( metric_value, float)): metrics[group_key]["metrics"][f"{mib}.{metric}"] = { "time": time.time(), "type": metric_type, "value": metric_value, "oid": oid, } if profile and profile not in metrics[group_key][ "profiles"]: metrics[group_key]["profiles"].append(profile) else: metrics[group_key]["fields"][f"{mib}.{metric}"] = { "time": time.time(), "type": metric_type, "value": metric_value, "oid": oid, } except: logger.exception( f"Exception processing data from {target} {varBind}") else: found, mib = self.is_mib_known(id, oid, target) if mib and mib not in remotemibs: remotemibs.append(mib) if found: retry = True break return retry, remotemibs, metrics
def get_engine(): mibBuilder = builder.MibBuilder() engine=SnmpEngine() builder_engine = engine.getMibBuilder() builder_engine.addMibSources(builder.DirMibSource('Scripts/python/snmp/mibs')) return engine