def _stampcheck(self, mib): stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) yield stampcheck.load() yield stampcheck.collect([mib.get_neighbors_last_change()]) defer.returnValue(stampcheck)
def _stampcheck(self, mib): stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) yield stampcheck.load() yield stampcheck.collect([mib.get_neighbors_last_change()]) defer.returnValue(stampcheck)
def test_invalid_serialized_value_should_be_interpreted_as_change(): ts = TimestampChecker(Mock(), Mock(), 'uptime') invalid_time_pickle = 'foobar' with patch('nav.models.manage.NetboxInfo.objects.get', return_value=invalid_time_pickle): yield ts.load() assert ts.is_changed()
def _get_stampcheck(self, mib): """Retrieves the last change timestamp of the LLDP remote table, returning a TimestampChecker instance reflecting it. """ stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) yield stampcheck.load() yield stampcheck.collect([mib.get_remote_last_change()]) defer.returnValue(stampcheck)
def _get_timestamps(self): stampcheck = TimestampChecker(self.agent, self.containers, "uptime") old_times = yield stampcheck.load() new_times = yield stampcheck.collect([]) changed = old_times and stampcheck.is_changed(COLDBOOT_MAX_DELTA) self._logger.debug("uptime data (changed=%s): %r", changed, new_times) timestamp, ticks = new_times[0] upsince = get_upsince(timestamp, ticks) self._logger.debug( "last sysuptime reset/rollover reported by device: %s", upsince) stampcheck.save() returnValue((changed, upsince))
def test_representation_inconsistencies_should_not_matter(): """Tests that loaded and collected timestamps can be compared, even if the internal and the persisted representations may differ between being tuples and lists. The internal representation was always with tuples, but the new JSON-based db persistence converts this to lists, since tuples don't exist in JSON. """ ts = TimestampChecker(Mock(), Mock(), 'sometime') json_data = "[[1559907627.0, 28245286]]" ts.collected_times = ((1559907627.0, 28245286), ) mock_info = Mock(value=json_data) with patch('nav.models.manage.NetboxInfo.objects.get', return_value=mock_info): yield ts.load() assert not ts.is_changed(max_deviation=60)
class Entity(Plugin): """Plugin to collect physical entity data from devices""" def __init__(self, *args, **kwargs): super(Entity, self).__init__(*args, **kwargs) self.alias_mapping = {} self.entitymib = EntityMib(self.agent) self.stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) @defer.inlineCallbacks def handle(self): self._logger.debug("Collecting physical entity data") need_to_collect = yield self._need_to_collect() #if need_to_collect: if True: physical_table = ( yield self.entitymib.get_entity_physical_table()) self._logger.debug("found %d entities", len(physical_table)) self._process_entities(physical_table) self.stampcheck.save() @defer.inlineCallbacks def _need_to_collect(self): yield self.stampcheck.load() yield self.stampcheck.collect([self.entitymib.get_last_change_time()]) result = yield self.stampcheck.is_changed() defer.returnValue(result) def _process_entities(self, result): """Process the list of collected entities.""" # be able to look up all entities using entPhysicalIndex entities = EntityTable(result) entities.clean_unicode() containers = [self._container_from_entity(entity) for entity in entities.values()] self._fix_hierarchy(containers) def _fix_hierarchy(self, containers): by_index = {c.index: c for c in containers} ghosts = set() for container in containers: if container.contained_in: parent_id = unicode(container.contained_in) parent = by_index.get(parent_id) if parent: container.contained_in = parent else: ghosts.add(str(container.contained_in)) container.contained_in = None if ghosts: self._logger.info( "kick your device vendor in the shin. entPhysicalContainedIn " "values refer to non-existant entities: %s", ", ".join(ghosts)) field_map = {k: 'entPhysical'+v for k, v in dict( index='Index', descr='Descr', vendor_type='VendorType', contained_in='ContainedIn', physical_class='Class', parent_relpos='ParentRelPos', name='Name', hardware_revision='HardwareRev', firmware_revision='FirmwareRev', software_revision='SoftwareRev', mfg_name='MfgName', model_name='ModelName', alias='Alias', asset_id='AssetID', fru='IsFRU', mfg_date='MfgDate', uris='Uris', serial='SerialNum', ).items()} class_map = {name: value for value, name in manage.NetboxEntity.CLASS_CHOICES} def _container_from_entity(self, ent): device_key = 'ENTITY-MIB:' + str(ent.get(0)) container = self.containers.factory(device_key, NetboxEntity) netbox = self.containers.factory(None, shadows.Netbox) container.netbox = netbox container.index = ent.get(0) container.source = 'ENTITY-MIB' for attr, column in self.field_map.items(): value = ent.get(column) if column == 'entPhysicalClass': value = self.class_map.get(value) if value is not None: setattr(container, attr, value) if getattr(container, 'serial', None): device = self.containers.factory(container.serial, shadows.Device) device.serial = container.serial for key in ('hardware', 'firmware', 'software'): val = getattr(container, key + '_revision') if val: setattr(device, key + '_version', val) device.active = True container.device = device return container
class Modules(Plugin): """Plugin to collect module data from devices""" def __init__(self, *args, **kwargs): super(Modules, self).__init__(*args, **kwargs) self.alias_mapping = {} self.entitymib = EntityMib(self.agent) self.stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) @defer.inlineCallbacks def handle(self): self._logger.debug("Collecting ENTITY-MIB module data") need_to_collect = yield self._need_to_collect() if need_to_collect: physical_table = (yield self.entitymib.get_entity_physical_table()) self.alias_mapping = yield self.entitymib.get_alias_mapping() self._process_entities(physical_table) self.stampcheck.save() @defer.inlineCallbacks def _need_to_collect(self): yield self.stampcheck.load() yield self.stampcheck.collect([self.entitymib.get_last_change_time()]) result = yield self.stampcheck.is_changed() defer.returnValue(result) def _device_from_entity(self, ent): serial_column = 'entPhysicalSerialNum' if serial_column in ent and ent[serial_column] and \ ent[serial_column].strip(): serial_number = ent[serial_column].strip() device_key = serial_number else: serial_number = None device_key = 'unknown-%s' % ent[0] device = self.containers.factory(device_key, shadows.Device) if serial_number: device.serial = serial_number if ent['entPhysicalHardwareRev']: device.hardware_version = ent['entPhysicalHardwareRev'].strip() if ent['entPhysicalSoftwareRev']: device.software_version = ent['entPhysicalSoftwareRev'].strip() if ent['entPhysicalFirmwareRev']: device.firmware_version = ent['entPhysicalFirmwareRev'].strip() device.active = True return device def _module_from_entity(self, ent): module = self.containers.factory(ent['entPhysicalSerialNum'], shadows.Module) netbox = self.containers.factory(None, shadows.Netbox) module.netbox = netbox module.model = ent['entPhysicalModelName'].strip() module.description = ent['entPhysicalDescr'].strip() module.name = ent['entPhysicalName'].strip() if module.name.strip().isdigit(): module.module_number = int(module.name.strip()) module.parent = None return module def _process_modules(self, entities): # map entity indexes to module containers module_containers = {} modules = entities.get_modules() for ent in modules: entity_index = ent[0] device = self._device_from_entity(ent) module = self._module_from_entity(ent) module.device = device module_containers[entity_index] = module self._logger.debug("module (entPhysIndex=%s): %r", entity_index, module) return module_containers def _process_ports(self, entities, module_containers): ports = entities.get_ports() netbox = self.containers.factory(None, shadows.Netbox) # Map interfaces to modules, if possible module_ifindex_map = {} # just for logging debug info for port in ports: entity_index = port[0] if entity_index in self.alias_mapping: module_entity = entities.get_nearest_module_parent(port) if module_entity and module_entity[0] in module_containers: module = module_containers[module_entity[0]] indices = self.alias_mapping[entity_index] for ifindex in indices: interface = self.containers.factory( ifindex, shadows.Interface) interface.netbox = netbox interface.ifindex = ifindex interface.module = module if module.name in module_ifindex_map: module_ifindex_map[module.name].append(ifindex) else: module_ifindex_map[module.name] = [ifindex] if module_ifindex_map: self._logger.debug("module/ifindex mapping: %r", module_ifindex_map) def _process_entities(self, result): """Process the list of collected entities.""" # be able to look up all entities using entPhysicalIndex entities = EntityTable(result) module_containers = self._process_modules(entities) self._process_ports(entities, module_containers)
class Modules(Plugin): """Plugin to collect module and chassis data from devices""" def __init__(self, *args, **kwargs): super(Modules, self).__init__(*args, **kwargs) self.alias_mapping = {} self.entitymib = EntityMib(self.agent) self.stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) @defer.inlineCallbacks def handle(self): self._logger.debug("Collecting ENTITY-MIB module data") need_to_collect = yield self._need_to_collect() if need_to_collect: physical_table = ( yield self.entitymib.get_useful_physical_table_columns()) alias_mapping = yield self.entitymib.retrieve_column( 'entAliasMappingIdentifier') self.alias_mapping = self._process_alias_mapping(alias_mapping) self._process_entities(physical_table) self.stampcheck.save() @defer.inlineCallbacks def _need_to_collect(self): yield self.stampcheck.load() yield self.stampcheck.collect([self.entitymib.get_last_change_time()]) result = yield self.stampcheck.is_changed() defer.returnValue(result) def _device_from_entity(self, ent, chassis=False): serial_column = 'entPhysicalSerialNum' if serial_column in ent and ent[serial_column] and \ ent[serial_column].strip(): serial_number = ent[serial_column].strip() device_key = serial_number else: serial_number = None device_key = 'unknown-%s' % ent[0] # check whether some plugin already registered a chassis device # without knowing its serial. If so, give the device two keys in the # container repository if chassis and self.containers.get(None, shadows.Device): device = self.containers.get(None, shadows.Device) self.containers[shadows.Device][device_key] = device else: device = self.containers.factory(device_key, shadows.Device) if serial_number: device.serial = serial_number if ent['entPhysicalHardwareRev']: device.hardware_version = ent['entPhysicalHardwareRev'].strip() if ent['entPhysicalSoftwareRev']: device.software_version = ent['entPhysicalSoftwareRev'].strip() if ent['entPhysicalFirmwareRev']: device.firmware_version = ent['entPhysicalFirmwareRev'].strip() device.active = True return device def _module_from_entity(self, ent): module = self.containers.factory(ent['entPhysicalSerialNum'], shadows.Module) netbox = self.containers.factory(None, shadows.Netbox) module.netbox = netbox module.model = ent['entPhysicalModelName'].strip() module.description = ent['entPhysicalDescr'].strip() module.name = ent['entPhysicalName'].strip() if module.name.strip().isdigit(): module.module_number = int(module.name.strip()) module.parent = None return module def _process_modules(self, entities): # map entity indexes to module containers module_containers = {} modules = entities.get_modules() for ent in modules: entity_index = ent[0] device = self._device_from_entity(ent) module = self._module_from_entity(ent) module.device = device module_containers[entity_index] = module self._logger.debug("module (entPhysIndex=%s): %r", entity_index, module) return module_containers def _process_chassis(self, entities): chassis = entities.get_chassis() if not chassis: self._logger.debug('No chassis found') return elif len(chassis) > 1: self._logger.debug('Found multiple chassis') # We don't really know how to handle a multiple chassis # situation. Best effort is to use the first one in the list. # This should be revised by someone who has stacked chassis # devices to test on. the_chassis = chassis[0] device = self._device_from_entity(the_chassis, chassis=True) netbox = self.containers.factory(None, shadows.Netbox) netbox.device = device def _process_ports(self, entities, module_containers): ports = entities.get_ports() netbox = self.containers.factory(None, shadows.Netbox) # Map interfaces to modules, if possible module_ifindex_map = {} #just for logging debug info for port in ports: entity_index = port[0] if entity_index in self.alias_mapping: module_entity = entities.get_nearest_module_parent(port) if module_entity and module_entity[0] in module_containers: module = module_containers[module_entity[0]] indices = self.alias_mapping[entity_index] for ifindex in indices: interface = self.containers.factory( ifindex, shadows.Interface) interface.netbox = netbox interface.ifindex = ifindex interface.module = module if module.name in module_ifindex_map: module_ifindex_map[module.name].append(ifindex) else: module_ifindex_map[module.name] = [ifindex] if module_ifindex_map: self._logger.debug("module/ifindex mapping: %r", module_ifindex_map) def _process_entities(self, result): """Process the list of collected entities.""" # be able to look up all entities using entPhysicalIndex entities = EntityTable(result) module_containers = self._process_modules(entities) self._process_chassis(entities) self._process_ports(entities, module_containers) def _process_alias_mapping(self, alias_mapping): mapping = {} for (phys_index, _logical), rowpointer in alias_mapping.items(): # Last element is ifindex. Preceding elements is an OID. ifindex = OID(rowpointer)[-1] if phys_index not in mapping: mapping[phys_index] = [] mapping[phys_index].append(ifindex) self._logger.debug("alias mapping: %r", mapping) return mapping
class Modules(Plugin): """Plugin to collect module and chassis data from devices""" def __init__(self, *args, **kwargs): super(Modules, self).__init__(*args, **kwargs) self.alias_mapping = {} self.entitymib = EntityMib(self.agent) self.stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME) @defer.inlineCallbacks def handle(self): self._logger.debug("Collecting ENTITY-MIB module data") need_to_collect = yield self._need_to_collect() if need_to_collect: physical_table = ( yield self.entitymib.get_useful_physical_table_columns()) alias_mapping = yield self.entitymib.retrieve_column( 'entAliasMappingIdentifier') self.alias_mapping = self._process_alias_mapping(alias_mapping) self._process_entities(physical_table) self.stampcheck.save() @defer.inlineCallbacks def _need_to_collect(self): yield self.stampcheck.load() yield self.stampcheck.collect([self.entitymib.get_last_change_time()]) result = yield self.stampcheck.is_changed() defer.returnValue(result) def _device_from_entity(self, ent, chassis=False): serial_column = 'entPhysicalSerialNum' if serial_column in ent and ent[serial_column] and \ ent[serial_column].strip(): serial_number = ent[serial_column].strip() device_key = serial_number else: serial_number = None device_key = 'unknown-%s' % ent[0] # check whether some plugin already registered a chassis device # without knowing its serial. If so, give the device two keys in the # container repository if chassis and self.containers.get(None, shadows.Device): device = self.containers.get(None, shadows.Device) self.containers[shadows.Device][device_key] = device else: device = self.containers.factory(device_key, shadows.Device) if serial_number: device.serial = serial_number if ent['entPhysicalHardwareRev']: device.hardware_version = ent['entPhysicalHardwareRev'].strip() if ent['entPhysicalSoftwareRev']: device.software_version = ent['entPhysicalSoftwareRev'].strip() if ent['entPhysicalFirmwareRev']: device.firmware_version = ent['entPhysicalFirmwareRev'].strip() device.active = True return device def _module_from_entity(self, ent): module = self.containers.factory(ent['entPhysicalSerialNum'], shadows.Module) netbox = self.containers.factory(None, shadows.Netbox) module.netbox = netbox module.model = ent['entPhysicalModelName'].strip() module.description = ent['entPhysicalDescr'].strip() module.name = ent['entPhysicalName'].strip() if module.name.strip().isdigit(): module.module_number = int(module.name.strip()) module.parent = None return module def _process_modules(self, entities): # map entity indexes to module containers module_containers = {} modules = entities.get_modules() for ent in modules: entity_index = ent[0] device = self._device_from_entity(ent) module = self._module_from_entity(ent) module.device = device module_containers[entity_index] = module self._logger.debug("module (entPhysIndex=%s): %r", entity_index, module) return module_containers def _process_chassis(self, entities): chassis = entities.get_chassis() if not chassis: self._logger.debug('No chassis found') return elif len(chassis) > 1: self._logger.debug('Found multiple chassis') # We don't really know how to handle a multiple chassis # situation. Best effort is to use the first one in the list. # This should be revised by someone who has stacked chassis # devices to test on. the_chassis = chassis[0] device = self._device_from_entity(the_chassis, chassis=True) netbox = self.containers.factory(None, shadows.Netbox) netbox.device = device def _process_ports(self, entities, module_containers): ports = entities.get_ports() netbox = self.containers.factory(None, shadows.Netbox) # Map interfaces to modules, if possible module_ifindex_map = {} #just for logging debug info for port in ports: entity_index = port[0] if entity_index in self.alias_mapping: module_entity = entities.get_nearest_module_parent(port) if module_entity and module_entity[0] in module_containers: module = module_containers[ module_entity[0] ] indices = self.alias_mapping[entity_index] for ifindex in indices: interface = self.containers.factory(ifindex, shadows.Interface) interface.netbox = netbox interface.ifindex = ifindex interface.module = module if module.name in module_ifindex_map: module_ifindex_map[module.name].append(ifindex) else: module_ifindex_map[module.name] = [ifindex] if module_ifindex_map: self._logger.debug("module/ifindex mapping: %r", module_ifindex_map) def _process_entities(self, result): """Process the list of collected entities.""" # be able to look up all entities using entPhysicalIndex entities = EntityTable(result) module_containers = self._process_modules(entities) self._process_chassis(entities) self._process_ports(entities, module_containers) def _process_alias_mapping(self, alias_mapping): mapping = {} for (phys_index, _logical), rowpointer in alias_mapping.items(): # Last element is ifindex. Preceding elements is an OID. ifindex = OID(rowpointer)[-1] if phys_index not in mapping: mapping[phys_index] = [] mapping[phys_index].append(ifindex) self._logger.debug("alias mapping: %r", mapping) return mapping