def _stampcheck(self, mib):
stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
yield stampcheck.load()
yield stampcheck.collect([mib.get_neighbors_last_change()])
defer.returnValue(stampcheck)
def _stampcheck(self, mib):
stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
yield stampcheck.load()
yield stampcheck.collect([mib.get_neighbors_last_change()])
defer.returnValue(stampcheck)
def test_invalid_serialized_value_should_be_interpreted_as_change():
ts = TimestampChecker(Mock(), Mock(), 'uptime')
invalid_time_pickle = 'foobar'
with patch('nav.models.manage.NetboxInfo.objects.get',
return_value=invalid_time_pickle):
yield ts.load()
assert ts.is_changed()
def _get_stampcheck(self, mib):
"""Retrieves the last change timestamp of the LLDP remote table, returning a
TimestampChecker instance reflecting it.
"""
stampcheck = TimestampChecker(self.agent, self.containers, INFO_VAR_NAME)
yield stampcheck.load()
yield stampcheck.collect([mib.get_remote_last_change()])
defer.returnValue(stampcheck)
def test_representation_inconsistencies_should_not_matter():
"""Tests that loaded and collected timestamps can be compared, even if the internal
and the persisted representations may differ between being tuples and lists.
The internal representation was always with tuples, but the new JSON-based db
persistence converts this to lists, since tuples don't exist in JSON.
"""
ts = TimestampChecker(Mock(), Mock(), 'sometime')
json_data = "[[1559907627.0, 28245286]]"
ts.collected_times = ((1559907627.0, 28245286), )
mock_info = Mock(value=json_data)
with patch('nav.models.manage.NetboxInfo.objects.get',
return_value=mock_info):
yield ts.load()
assert not ts.is_changed(max_deviation=60)
def _get_timestamps(self):
stampcheck = TimestampChecker(self.agent, self.containers, "uptime")
old_times = yield stampcheck.load()
new_times = yield stampcheck.collect([])
changed = old_times and stampcheck.is_changed(COLDBOOT_MAX_DELTA)
self._logger.debug("uptime data (changed=%s): %r", changed, new_times)
timestamp, ticks = new_times[0]
upsince = get_upsince(timestamp, ticks)
self._logger.debug(
"last sysuptime reset/rollover reported by device: %s", upsince)
stampcheck.save()
returnValue((changed, upsince))
def __init__(self, *args, **kwargs):
super(Entity, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
class Entity(Plugin):
"""Plugin to collect physical entity data from devices"""
def __init__(self, *args, **kwargs):
super(Entity, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
@defer.inlineCallbacks
def handle(self):
self._logger.debug("Collecting physical entity data")
need_to_collect = yield self._need_to_collect()
#if need_to_collect:
if True:
physical_table = (
yield self.entitymib.get_entity_physical_table())
self._logger.debug("found %d entities", len(physical_table))
self._process_entities(physical_table)
self.stampcheck.save()
@defer.inlineCallbacks
def _need_to_collect(self):
yield self.stampcheck.load()
yield self.stampcheck.collect([self.entitymib.get_last_change_time()])
result = yield self.stampcheck.is_changed()
defer.returnValue(result)
def _process_entities(self, result):
"""Process the list of collected entities."""
# be able to look up all entities using entPhysicalIndex
entities = EntityTable(result)
entities.clean_unicode()
containers = [self._container_from_entity(entity)
for entity in entities.values()]
self._fix_hierarchy(containers)
def _fix_hierarchy(self, containers):
by_index = {c.index: c for c in containers}
ghosts = set()
for container in containers:
if container.contained_in:
parent_id = unicode(container.contained_in)
parent = by_index.get(parent_id)
if parent:
container.contained_in = parent
else:
ghosts.add(str(container.contained_in))
container.contained_in = None
if ghosts:
self._logger.info(
"kick your device vendor in the shin. entPhysicalContainedIn "
"values refer to non-existant entities: %s", ", ".join(ghosts))
field_map = {k: 'entPhysical'+v for k, v in dict(
index='Index',
descr='Descr',
vendor_type='VendorType',
contained_in='ContainedIn',
physical_class='Class',
parent_relpos='ParentRelPos',
name='Name',
hardware_revision='HardwareRev',
firmware_revision='FirmwareRev',
software_revision='SoftwareRev',
mfg_name='MfgName',
model_name='ModelName',
alias='Alias',
asset_id='AssetID',
fru='IsFRU',
mfg_date='MfgDate',
uris='Uris',
serial='SerialNum',
).items()}
class_map = {name: value
for value, name in manage.NetboxEntity.CLASS_CHOICES}
def _container_from_entity(self, ent):
device_key = 'ENTITY-MIB:' + str(ent.get(0))
container = self.containers.factory(device_key, NetboxEntity)
netbox = self.containers.factory(None, shadows.Netbox)
container.netbox = netbox
container.index = ent.get(0)
container.source = 'ENTITY-MIB'
for attr, column in self.field_map.items():
value = ent.get(column)
if column == 'entPhysicalClass':
value = self.class_map.get(value)
if value is not None:
setattr(container, attr, value)
if getattr(container, 'serial', None):
device = self.containers.factory(container.serial, shadows.Device)
device.serial = container.serial
for key in ('hardware', 'firmware', 'software'):
val = getattr(container, key + '_revision')
if val:
setattr(device, key + '_version', val)
device.active = True
container.device = device
return container
class Modules(Plugin):
"""Plugin to collect module data from devices"""
def __init__(self, *args, **kwargs):
super(Modules, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
@defer.inlineCallbacks
def handle(self):
self._logger.debug("Collecting ENTITY-MIB module data")
need_to_collect = yield self._need_to_collect()
if need_to_collect:
physical_table = (yield self.entitymib.get_entity_physical_table())
self.alias_mapping = yield self.entitymib.get_alias_mapping()
self._process_entities(physical_table)
self.stampcheck.save()
@defer.inlineCallbacks
def _need_to_collect(self):
yield self.stampcheck.load()
yield self.stampcheck.collect([self.entitymib.get_last_change_time()])
result = yield self.stampcheck.is_changed()
defer.returnValue(result)
def _device_from_entity(self, ent):
serial_column = 'entPhysicalSerialNum'
if serial_column in ent and ent[serial_column] and \
ent[serial_column].strip():
serial_number = ent[serial_column].strip()
device_key = serial_number
else:
serial_number = None
device_key = 'unknown-%s' % ent[0]
device = self.containers.factory(device_key, shadows.Device)
if serial_number:
device.serial = serial_number
if ent['entPhysicalHardwareRev']:
device.hardware_version = ent['entPhysicalHardwareRev'].strip()
if ent['entPhysicalSoftwareRev']:
device.software_version = ent['entPhysicalSoftwareRev'].strip()
if ent['entPhysicalFirmwareRev']:
device.firmware_version = ent['entPhysicalFirmwareRev'].strip()
device.active = True
return device
def _module_from_entity(self, ent):
module = self.containers.factory(ent['entPhysicalSerialNum'],
shadows.Module)
netbox = self.containers.factory(None, shadows.Netbox)
module.netbox = netbox
module.model = ent['entPhysicalModelName'].strip()
module.description = ent['entPhysicalDescr'].strip()
module.name = ent['entPhysicalName'].strip()
if module.name.strip().isdigit():
module.module_number = int(module.name.strip())
module.parent = None
return module
def _process_modules(self, entities):
# map entity indexes to module containers
module_containers = {}
modules = entities.get_modules()
for ent in modules:
entity_index = ent[0]
device = self._device_from_entity(ent)
module = self._module_from_entity(ent)
module.device = device
module_containers[entity_index] = module
self._logger.debug("module (entPhysIndex=%s): %r", entity_index,
module)
return module_containers
def _process_ports(self, entities, module_containers):
ports = entities.get_ports()
netbox = self.containers.factory(None, shadows.Netbox)
# Map interfaces to modules, if possible
module_ifindex_map = {} # just for logging debug info
for port in ports:
entity_index = port[0]
if entity_index in self.alias_mapping:
module_entity = entities.get_nearest_module_parent(port)
if module_entity and module_entity[0] in module_containers:
module = module_containers[module_entity[0]]
indices = self.alias_mapping[entity_index]
for ifindex in indices:
interface = self.containers.factory(
ifindex, shadows.Interface)
interface.netbox = netbox
interface.ifindex = ifindex
interface.module = module
if module.name in module_ifindex_map:
module_ifindex_map[module.name].append(ifindex)
else:
module_ifindex_map[module.name] = [ifindex]
if module_ifindex_map:
self._logger.debug("module/ifindex mapping: %r",
module_ifindex_map)
def _process_entities(self, result):
"""Process the list of collected entities."""
# be able to look up all entities using entPhysicalIndex
entities = EntityTable(result)
module_containers = self._process_modules(entities)
self._process_ports(entities, module_containers)
class Modules(Plugin):
"""Plugin to collect module and chassis data from devices"""
def __init__(self, *args, **kwargs):
super(Modules, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
@defer.inlineCallbacks
def handle(self):
self._logger.debug("Collecting ENTITY-MIB module data")
need_to_collect = yield self._need_to_collect()
if need_to_collect:
physical_table = (
yield self.entitymib.get_useful_physical_table_columns())
alias_mapping = yield self.entitymib.retrieve_column(
'entAliasMappingIdentifier')
self.alias_mapping = self._process_alias_mapping(alias_mapping)
self._process_entities(physical_table)
self.stampcheck.save()
@defer.inlineCallbacks
def _need_to_collect(self):
yield self.stampcheck.load()
yield self.stampcheck.collect([self.entitymib.get_last_change_time()])
result = yield self.stampcheck.is_changed()
defer.returnValue(result)
def _device_from_entity(self, ent, chassis=False):
serial_column = 'entPhysicalSerialNum'
if serial_column in ent and ent[serial_column] and \
ent[serial_column].strip():
serial_number = ent[serial_column].strip()
device_key = serial_number
else:
serial_number = None
device_key = 'unknown-%s' % ent[0]
# check whether some plugin already registered a chassis device
# without knowing its serial. If so, give the device two keys in the
# container repository
if chassis and self.containers.get(None, shadows.Device):
device = self.containers.get(None, shadows.Device)
self.containers[shadows.Device][device_key] = device
else:
device = self.containers.factory(device_key, shadows.Device)
if serial_number:
device.serial = serial_number
if ent['entPhysicalHardwareRev']:
device.hardware_version = ent['entPhysicalHardwareRev'].strip()
if ent['entPhysicalSoftwareRev']:
device.software_version = ent['entPhysicalSoftwareRev'].strip()
if ent['entPhysicalFirmwareRev']:
device.firmware_version = ent['entPhysicalFirmwareRev'].strip()
device.active = True
return device
def _module_from_entity(self, ent):
module = self.containers.factory(ent['entPhysicalSerialNum'],
shadows.Module)
netbox = self.containers.factory(None, shadows.Netbox)
module.netbox = netbox
module.model = ent['entPhysicalModelName'].strip()
module.description = ent['entPhysicalDescr'].strip()
module.name = ent['entPhysicalName'].strip()
if module.name.strip().isdigit():
module.module_number = int(module.name.strip())
module.parent = None
return module
def _process_modules(self, entities):
# map entity indexes to module containers
module_containers = {}
modules = entities.get_modules()
for ent in modules:
entity_index = ent[0]
device = self._device_from_entity(ent)
module = self._module_from_entity(ent)
module.device = device
module_containers[entity_index] = module
self._logger.debug("module (entPhysIndex=%s): %r", entity_index,
module)
return module_containers
def _process_chassis(self, entities):
chassis = entities.get_chassis()
if not chassis:
self._logger.debug('No chassis found')
return
elif len(chassis) > 1:
self._logger.debug('Found multiple chassis')
# We don't really know how to handle a multiple chassis
# situation. Best effort is to use the first one in the list.
# This should be revised by someone who has stacked chassis
# devices to test on.
the_chassis = chassis[0]
device = self._device_from_entity(the_chassis, chassis=True)
netbox = self.containers.factory(None, shadows.Netbox)
netbox.device = device
def _process_ports(self, entities, module_containers):
ports = entities.get_ports()
netbox = self.containers.factory(None, shadows.Netbox)
# Map interfaces to modules, if possible
module_ifindex_map = {} #just for logging debug info
for port in ports:
entity_index = port[0]
if entity_index in self.alias_mapping:
module_entity = entities.get_nearest_module_parent(port)
if module_entity and module_entity[0] in module_containers:
module = module_containers[module_entity[0]]
indices = self.alias_mapping[entity_index]
for ifindex in indices:
interface = self.containers.factory(
ifindex, shadows.Interface)
interface.netbox = netbox
interface.ifindex = ifindex
interface.module = module
if module.name in module_ifindex_map:
module_ifindex_map[module.name].append(ifindex)
else:
module_ifindex_map[module.name] = [ifindex]
if module_ifindex_map:
self._logger.debug("module/ifindex mapping: %r",
module_ifindex_map)
def _process_entities(self, result):
"""Process the list of collected entities."""
# be able to look up all entities using entPhysicalIndex
entities = EntityTable(result)
module_containers = self._process_modules(entities)
self._process_chassis(entities)
self._process_ports(entities, module_containers)
def _process_alias_mapping(self, alias_mapping):
mapping = {}
for (phys_index, _logical), rowpointer in alias_mapping.items():
# Last element is ifindex. Preceding elements is an OID.
ifindex = OID(rowpointer)[-1]
if phys_index not in mapping:
mapping[phys_index] = []
mapping[phys_index].append(ifindex)
self._logger.debug("alias mapping: %r", mapping)
return mapping
def test_is_changed(loaded, collected, max_deviation, expected, description):
ts = TimestampChecker(Mock(), Mock(), 'sometime')
ts.loaded_times = [loaded] if loaded else None
ts.collected_times = [collected]
assert ts.is_changed(max_deviation=max_deviation) == expected, description
def __init__(self, *args, **kwargs):
super(Modules, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
class Modules(Plugin):
"""Plugin to collect module and chassis data from devices"""
def __init__(self, *args, **kwargs):
super(Modules, self).__init__(*args, **kwargs)
self.alias_mapping = {}
self.entitymib = EntityMib(self.agent)
self.stampcheck = TimestampChecker(self.agent, self.containers,
INFO_VAR_NAME)
@defer.inlineCallbacks
def handle(self):
self._logger.debug("Collecting ENTITY-MIB module data")
need_to_collect = yield self._need_to_collect()
if need_to_collect:
physical_table = (
yield self.entitymib.get_useful_physical_table_columns())
alias_mapping = yield self.entitymib.retrieve_column(
'entAliasMappingIdentifier')
self.alias_mapping = self._process_alias_mapping(alias_mapping)
self._process_entities(physical_table)
self.stampcheck.save()
@defer.inlineCallbacks
def _need_to_collect(self):
yield self.stampcheck.load()
yield self.stampcheck.collect([self.entitymib.get_last_change_time()])
result = yield self.stampcheck.is_changed()
defer.returnValue(result)
def _device_from_entity(self, ent, chassis=False):
serial_column = 'entPhysicalSerialNum'
if serial_column in ent and ent[serial_column] and \
ent[serial_column].strip():
serial_number = ent[serial_column].strip()
device_key = serial_number
else:
serial_number = None
device_key = 'unknown-%s' % ent[0]
# check whether some plugin already registered a chassis device
# without knowing its serial. If so, give the device two keys in the
# container repository
if chassis and self.containers.get(None, shadows.Device):
device = self.containers.get(None, shadows.Device)
self.containers[shadows.Device][device_key] = device
else:
device = self.containers.factory(device_key, shadows.Device)
if serial_number:
device.serial = serial_number
if ent['entPhysicalHardwareRev']:
device.hardware_version = ent['entPhysicalHardwareRev'].strip()
if ent['entPhysicalSoftwareRev']:
device.software_version = ent['entPhysicalSoftwareRev'].strip()
if ent['entPhysicalFirmwareRev']:
device.firmware_version = ent['entPhysicalFirmwareRev'].strip()
device.active = True
return device
def _module_from_entity(self, ent):
module = self.containers.factory(ent['entPhysicalSerialNum'],
shadows.Module)
netbox = self.containers.factory(None, shadows.Netbox)
module.netbox = netbox
module.model = ent['entPhysicalModelName'].strip()
module.description = ent['entPhysicalDescr'].strip()
module.name = ent['entPhysicalName'].strip()
if module.name.strip().isdigit():
module.module_number = int(module.name.strip())
module.parent = None
return module
def _process_modules(self, entities):
# map entity indexes to module containers
module_containers = {}
modules = entities.get_modules()
for ent in modules:
entity_index = ent[0]
device = self._device_from_entity(ent)
module = self._module_from_entity(ent)
module.device = device
module_containers[entity_index] = module
self._logger.debug("module (entPhysIndex=%s): %r",
entity_index, module)
return module_containers
def _process_chassis(self, entities):
chassis = entities.get_chassis()
if not chassis:
self._logger.debug('No chassis found')
return
elif len(chassis) > 1:
self._logger.debug('Found multiple chassis')
# We don't really know how to handle a multiple chassis
# situation. Best effort is to use the first one in the list.
# This should be revised by someone who has stacked chassis
# devices to test on.
the_chassis = chassis[0]
device = self._device_from_entity(the_chassis, chassis=True)
netbox = self.containers.factory(None, shadows.Netbox)
netbox.device = device
def _process_ports(self, entities, module_containers):
ports = entities.get_ports()
netbox = self.containers.factory(None, shadows.Netbox)
# Map interfaces to modules, if possible
module_ifindex_map = {} #just for logging debug info
for port in ports:
entity_index = port[0]
if entity_index in self.alias_mapping:
module_entity = entities.get_nearest_module_parent(port)
if module_entity and module_entity[0] in module_containers:
module = module_containers[ module_entity[0] ]
indices = self.alias_mapping[entity_index]
for ifindex in indices:
interface = self.containers.factory(ifindex,
shadows.Interface)
interface.netbox = netbox
interface.ifindex = ifindex
interface.module = module
if module.name in module_ifindex_map:
module_ifindex_map[module.name].append(ifindex)
else:
module_ifindex_map[module.name] = [ifindex]
if module_ifindex_map:
self._logger.debug("module/ifindex mapping: %r",
module_ifindex_map)
def _process_entities(self, result):
"""Process the list of collected entities."""
# be able to look up all entities using entPhysicalIndex
entities = EntityTable(result)
module_containers = self._process_modules(entities)
self._process_chassis(entities)
self._process_ports(entities, module_containers)
def _process_alias_mapping(self, alias_mapping):
mapping = {}
for (phys_index, _logical), rowpointer in alias_mapping.items():
# Last element is ifindex. Preceding elements is an OID.
ifindex = OID(rowpointer)[-1]
if phys_index not in mapping:
mapping[phys_index] = []
mapping[phys_index].append(ifindex)
self._logger.debug("alias mapping: %r", mapping)
return mapping