def _common_data_cleaner(self, processed_data, raw_data): devtype = self._get_devtype_from_input(raw_data) for i, entry in enumerate(processed_data): if not entry: continue if entry.get('_chassisType', '') == 'Mac Address': entry['peerHostname'] = convert_macaddr_format_to_colon( entry.get('peerHostname', '0000.0000.0000')) if 'peerIfname' in entry: subtype = entry.get('subtype', '') if subtype == 'Mac Address': entry['subtype'] = 'macddress' entry['peerMacaddr'] = convert_macaddr_format_to_colon( entry.get('peerIfname', '0000.0000.0000')) entry['peerIfname'] = '-' entry['peerIfindex'] = 0 elif subtype == 'ifindex': entry['subtype'] = 'ifindex' entry['peerIfname'] = '-' entry['peerMacaddr'] = '00:00:00:00:00:00' else: entry['subtype'] = 'ifname' entry['peerMacaddr'] = '00:00:00:00:00:00' entry['peerIfindex'] = 0 if devtype == 'nxos': entry['peerHostname'] = re.sub(r'\(.*\)', '', entry['peerHostname']) entry['ifname'] = re.sub(r'^Eth?(\d)', 'Ethernet\g<1>', entry['ifname']) return processed_data
def _clean_eos_data(self, processed_data, raw_data): for entry in processed_data: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) if ',' in entry['oif']: entry['oif'] = entry['oif'].split(',')[0].strip() return processed_data
def _clean_eos_data(self, processed_data, raw_data): for entry in processed_data: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) # EOS has entries with OIF of type: "Vlan4094, Port-Channel1" # We need only the first part, we pick up the second from the # MAC table if ',' in entry['oif']: entry['oif'] = entry['oif'].split(',')[0].strip() return processed_data
def _clean_eos_data(self, processed_data, raw_data): foo = 0 for entry in processed_data: if '.' in entry['macaddr']: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) entry['oif'] = expand_eos_ifname(entry['oif']) self._add_mackey_protocol(entry) return processed_data
def _clean_eos_data(self, processed_data, raw_data): for entry in processed_data: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) vtepIP = re.match(r'(\S+)\(([0-9.]+)\)', entry['oif']) if vtepIP: entry['remoteVtepIp'] = vtepIP.group(2) entry['oif'] = vtepIP.group(1) entry['flags'] = 'remote' self._add_mackey(entry) return processed_data
def _clean_nxos_data(self, processed_data, raw_data): drop_indices = [] for i, entry in enumerate(processed_data): if not entry['ipAddress']: drop_indices.append(i) continue entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) processed_data = np.delete(processed_data, drop_indices).tolist() return processed_data
def _clean_iosxr_data(self, processed_data, raw_data): for entry in processed_data: if entry['macaddr'] is None: entry['state'] = "failed" entry['macaddr'] = '00:00:00:00:00:00' else: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) entry['state'] = "reachable" if ':' in entry['ipAddress']: # We need to fix up the interface name for IPv6 ND entriie entry['oif'] = iosxr_get_full_ifname(entry.get('oif', '')) return processed_data
def _clean_eos_data(self, processed_data, raw_data): drop_indices = [] for i, entry in enumerate(processed_data): entry['oif'] = expand_eos_ifname(entry['oif']) if entry['oif'].startswith( 'Vxlan') and 'remoteVtepIp' not in entry: drop_indices.append(i) continue if '.' in entry['macaddr']: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) self._add_mackey_protocol(entry) processed_data = np.delete(processed_data, drop_indices).tolist() return processed_data
def _clean_nxos_data(self, processed_data, raw_data): for entry in processed_data: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) vtepIP = re.match(r'(\S+)\(([0-9.]+)\)', entry['oif']) if vtepIP: entry['remoteVtepIp'] = vtepIP.group(2) entry['oif'] = vtepIP.group(1) entry['flags'] = 'remote' else: entry['oif'] = expand_nxos_ifname(entry['oif']) entry['remoteVtepIp'] = '' if entry.get('vlan', '-') == '-': entry['vlan'] = 0 self._add_mackey_protocol(entry) return processed_data
def _clean_nxos_data(self, processed_data, raw_data): for entry in processed_data: entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) vtepIP = re.match(r'(\S+)\(([0-9.]+)\)', entry['oif']) if vtepIP: entry['remoteVtepIp'] = vtepIP.group(2) entry['oif'] = vtepIP.group(1) entry['flags'] = 'remote' else: if entry['oif'].startswith('Eth'): entry['oif'] = entry['oif'].replace('Eth', 'Ethernet') elif entry['oif'].startswith('Po'): entry['oif'] = entry['oif'].replace('Po', 'port-channel') self._add_mackey_protocol(entry) return processed_data
def _clean_nxos_data(self, processed_data, raw_data): """Complex cleanup of NXOS interface data""" new_entries = [] unnum_intf = {} drop_indices = [] entry_dict = defaultdict(dict) unnum_intf_entry_idx = [] # backtrack to interface to fix for entry_idx, entry in enumerate(processed_data): # if its the Linux ip link command output, massage the ifname # and copy over the values and drop the entry_dict if entry.get('_entryType', '') == 'mtumac': old_entry = entry_dict[entry['ifname']] if old_entry: old_entry['mtu'] = entry.get('mtu', 0) old_entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) if old_entry.get('_portchannel'): pc_entry = entry_dict.get(old_entry['master'], None) if pc_entry: # The later ethernet port entries' MAC addr wins # over the earlier ones pc_entry['mtu'] = entry.get('mtu', 0) pc_entry['macaddr'] = entry['macaddr'] drop_indices.append(entry_idx) continue entry_dict[entry['ifname']] = entry # artificial field for comparison with previous poll result entry["statusChangeTimestamp1"] = entry.get( "statusChangeTimestamp", '') entry['state'] = entry.get('state', 'unknown').lower() if entry.get('vrf', 'default') != 'default': entry['master'] = entry['vrf'] else: entry['master'] = '' if 'routeDistinguisher' in entry: entry['macaddr'] = "00:00:00:00:00:00" entry['adminState'] = entry.get("state", "up").lower() continue if 'reason' in entry: if entry['reason'] == '"Administratively down': entry['adminState'] = 'down' else: entry['adminState'] = 'up' if entry['reason'] is not None: entry['reason'] = entry['reason'].lower() if entry['reason'] == 'none' or not entry['reason']: entry['reason'] = '' portmode = entry.get('_portmode', '') if portmode == 'access' or portmode == 'trunk': entry['master'] = 'bridge' portchan = entry.get('_portchannel', '') if portchan: entry['master'] = f'port-channel{portchan}' entry['type'] = 'bond_slave' if entry['ifname'].startswith('port-channel'): entry['type'] = 'bond' if 'ipAddressList' in entry: pri_ipaddr = f"{entry['ipAddressList']}/{entry['_maskLen']}" ipaddr = [pri_ipaddr] for i, elem in enumerate(entry.get('_secIPs', [])): if elem: ipaddr.append(f"{elem}/{entry['_secmasklens'][i]}") entry['ipAddressList'] = ipaddr else: entry['ipAddressList'] = [] if 'ip6AddressList' in entry: if '_linklocal' in entry: entry['ip6AddressList'].append(entry['_linklocal']) else: entry['ip6AddressList'] = [] if entry.get('_anycastMac', ''): entry['interfaceMac'] = entry.get('macaddr', '') entry['macaddr'] = entry['_anycastMac'] if entry.get('_forwardMode', '') != 'Anycast': entry['reason'] += ', Fabric forwarding mode not enabled' entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) if entry.get('_unnum_intf', ''): if entry['ifname'] in unnum_intf: # IPv6 has link local, so unnumbered is a v4 construct entry['ipAddressList'] = [unnum_intf[entry['ifname']]] else: unnum_intf_entry_idx.append(entry_idx) for elem in entry.get('_child_intf', []): unnum_intf[elem] = [pri_ipaddr] speed = entry.get('speed', '') if isinstance(speed, str): if speed.startswith("unknown enum") or (speed == "auto"): speed = '0' elif speed.startswith('a-'): speed = speed[2:] if speed.endswith('G'): speed = int(speed[:-1])*1000 else: speed = int(speed) entry['speed'] = speed entry['type'] = entry.get('type', '').lower() if entry['type'] == 'eth': entry['type'] = 'ethernet' if entry['ifname'].startswith('Vlan'): entry['type'] = 'vlan' elif entry['ifname'].startswith('nve'): entry['type'] = 'vxlan' entry['master'] = 'bridge' elif entry['ifname'].startswith('loopback'): entry['type'] = 'loopback' if entry['type'] == 'vlan' and entry['ifname'].startswith('Vlan'): entry['vlan'] = int(entry['ifname'].split('Vlan')[1]) # have this at the end to avoid messing up processing # Fix unnumbered interface references for idx in unnum_intf_entry_idx: entry = processed_data[idx] entry['ipAddressList'] = unnum_intf.get(entry['ifname'], []) if drop_indices: processed_data = np.delete(processed_data, drop_indices).tolist() if new_entries: processed_data.extend(new_entries) return processed_data
def _clean_nxos_data(self, processed_data, raw_data): """Complex cleanup of NXOS interface data""" new_entries = [] created_if_list = set() unnum_intf = {} add_bridge_intf = False bridge_intf_state = "down" bridge_mtu = 1500 unnum_intf_entry_idx = [] # backtrack to interface to fix for entry_idx, entry in enumerate(processed_data): entry['origIfname'] = entry['ifname'] # artificial field for comparison with previous poll result entry["statusChangeTimestamp1"] = entry.get( "statusChangeTimestamp", '') if entry['type'] == 'eth': entry['type'] = 'ethernet' if ('vrf' in entry and entry['vrf'] != 'default' and entry['vrf'] not in created_if_list): # Our normalized behavior is to treat VRF as an interface # NXOS doesn't do that. So, create a dummy interface new_entry = { 'ifname': entry['vrf'], 'mtu': 65536, 'state': 'up', 'type': 'vrf', 'master': '', 'vlan': 0 } new_entries.append(new_entry) created_if_list.add(entry['vrf']) entry['master'] = entry['vrf'] else: entry['master'] = '' if entry['_portmode'] == 'access' or entry['_portmode'] == 'trunk': entry['master'] = 'bridge' add_bridge_intf = True if entry['state'] == "up": bridge_intf_state = "up" if entry.get('mtu', 1500) < bridge_mtu: bridge_mtu = entry.get('mtu', 0) if 'ipAddressList' in entry: pri_ipaddr = f"{entry['ipAddressList']}/{entry['_maskLen']}" ipaddr = [pri_ipaddr] for i, elem in enumerate(entry.get('_secIPs', [])): if elem: ipaddr.append(f"{elem}/{entry['_secmasklens'][i]}") entry['ipAddressList'] = ipaddr if 'ip6AddressList' in entry: if '_linklocal' in entry: entry['ip6AddressList'].append(entry['_linklocal']) entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) if entry.get('_unnum_intf', ''): if entry['ifname'] in unnum_intf: # IPv6 has link local, so unnumbered is a v4 construct entry['ipAddressList'] = [unnum_intf[entry['ifname']]] else: unnum_intf_entry_idx.append(entry_idx) for elem in entry.get('_child_intf', []): unnum_intf[elem] = [pri_ipaddr] speed = entry.get('speed', '') if isinstance(speed, str) and speed.startswith("unknown enum"): entry['speed'] = 0 entry['type'] = entry['type'].lower() if entry['ifname'].startswith('Vlan'): entry['type'] = 'vlan' elif entry['ifname'].startswith('nve'): entry['type'] = 'vxlan' entry['master'] = 'bridge' elif entry['ifname'].startswith('loopback'): entry['type'] = 'loopback' if entry['type'] == 'vlan' and entry['ifname'].startswith('Vlan'): entry['vlan'] = int(entry['ifname'].split('Vlan')[1]) # have this at the end to avoid messing up processing entry['ifname'] = entry['ifname'].replace('/', '-') # Fix unnumbered interface references for idx in unnum_intf_entry_idx: entry = processed_data[idx] entry['ipAddressList'] = unnum_intf.get(entry['origIfname'], []) # Add bridge interface if add_bridge_intf: new_entry = { 'ifname': 'bridge', 'mtu': bridge_mtu, 'state': bridge_intf_state, 'type': 'bridge', 'master': '', 'vlan': 0 } new_entries.append(new_entry) if new_entries: processed_data.extend(new_entries) return processed_data
def _clean_iosxr_data(self, processed_data, raw_data): entry_dict = {} for i, entry in enumerate(processed_data): if entry.get('_entryType', '') == 'vrf': entry['ifname'] = entry['vrf'] entry['master'] = '' entry['type'] = 'vrf' entry['state'] = entry['adminState'] = 'up' continue type = entry.get('type', 'ethernet').lower() if 'aggregated ethernet' in type: type = 'bond' elif 'ethernet' in type: type = 'ethernet' entry['type'] = type bondMbrs = entry.get('_bondMbrs', []) or [] if type == 'bond' and bondMbrs: for mbr in bondMbrs: if mbr in entry_dict: mbr_entry = entry_dict[mbr] mbr_entry['type'] = 'bond_slave' mbr_entry['master'] = entry['ifname'] else: entry_dict[mbr] = { 'master': entry['ifname'], 'type': 'bond_slave' } if entry['state'] == 'administratively down': entry['state'] = 'down' entry['adminState'] = 'down' entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) entry['interfaceMac'] = convert_macaddr_format_to_colon( entry.get('interfaceMac', '0000.0000.0000')) if entry.get('vlan', '') and entry.get('innerVlan', ''): entry['type'] = "qinq" lastChange = parse(entry.get('statusChangeTimestamp', ''), settings={ 'RELATIVE_BASE': datetime.fromtimestamp( (raw_data[0]['timestamp']) / 1000), }) if lastChange: entry['statusChangeTimestamp'] = int(lastChange.timestamp() * 1000) if 'ipAddressList' not in entry: entry['ipAddressList'] = [] if 'ip6AddressList' not in entry: entry['ip6AddressList'] = [] if entry['ipAddressList'] or entry['ip6AddressList']: entry['master'] = entry.get('vrf', '') if entry['ifname'] in entry_dict: add_info = entry_dict[entry['ifname']] entry['master'] = add_info.get('master', '') entry['type'] = add_info.get('type', '') else: entry_dict[entry['ifname']] = entry return processed_data
def _clean_nxos_data(self, processed_data, raw_data): """Complex cleanup of NXOS interface data""" def fix_nxos_speed(speed: str): if isinstance(speed, str): speed = speed.strip() if speed.startswith("unknown enum") or (speed == "auto"): speed = '0' elif speed.startswith('a-'): speed = speed[2:] if speed.endswith('G'): speed = int(speed[:-1]) * 1000 elif speed.endswith('Kbit'): speed = int(speed.split()[0]) / 1000 elif speed: speed = int(speed) else: speed = 0 return speed new_entries = [] unnum_intf = {} drop_indices = [] entry_dict = defaultdict(dict) unnum_intf_entry_idx = [] # backtrack to interface to fix for entry_idx, entry in enumerate(processed_data): # if its the Linux ip link command output, massage the ifname # and copy over the values and drop the entry_dict if entry.get('_entryType', '') == 'mtumac': old_entry = entry_dict[entry['ifname']] if old_entry: if entry['adminState']: old_entry['adminState'] = entry['adminState'] old_entry['mtu'] = entry.get('mtu', 0) or old_entry['mtu'] macaddr = entry.get('macaddr', '') if macaddr and not old_entry.get('_anycastMac', ''): # VLAN MAC addr MUST not be replaced due to fabric SVI old_entry['macaddr'] = convert_macaddr_format_to_colon( macaddr) old_entry['numChanges'] = entry['numChanges'] or 0 if entry.get('speed', ''): old_entry['speed'] = fix_nxos_speed(entry.get('speed')) lastChange = entry.get('statusChangeTimestamp', '') if lastChange: if any(x in lastChange for x in 'dwmy'): lastChange = f'{lastChange} hours ago' lastChange = parse( lastChange, settings={ 'RELATIVE_BASE': datetime.fromtimestamp( (raw_data[0]['timestamp']) / 1000), 'TIMEZONE': 'UTC' }) if lastChange: old_entry['statusChangeTimestamp'] = int( lastChange.timestamp() * 1000) else: entry['statusChangeTimestamp'] = 0 drop_indices.append(entry_idx) continue entry_dict[entry['ifname']] = entry entry["statusChangeTimestamp1"] = entry.get( "statusChangeTimestamp", '') if entry.get('vrf', ''): entry['master'] = entry['vrf'] if 'routeDistinguisher' in entry: # This is a VRF entry entry['macaddr'] = "00:00:00:00:00:00" entry['adminState'] = entry.get("state", "up").lower() entry['state'] = entry.get('state', 'down').lower() continue if 'reason' in entry: if entry['reason'] is not None: entry['reason'] = entry['reason'].lower() if entry['reason'] == 'none' or not entry['reason']: entry['reason'] = '' if entry['reason'] in [ "link not connected", "xcvr not inserted" ]: entry['state'] = 'notConnected' portmode = entry.get('_portmode', '') if portmode == 'access' or portmode == 'trunk': entry['master'] = 'bridge' portchan = entry.get('_portchannel', 0) if portchan: entry['master'] = f'port-channel{portchan}' entry['type'] = 'bond_slave' if entry['ifname'].startswith('port-channel'): entry['type'] = 'bond' if not entry.get('macaddr', ''): entry['macaddr'] = "00:00:00:00:00:00" if entry.get('ipAddressList', None): pri_ipaddr = f"{entry['ipAddressList']}/{entry['_maskLen']}" ipaddr = [pri_ipaddr] for i, elem in enumerate(entry.get('_secIPs', [])): if elem: ipaddr.append(f"{elem}/{entry['_secmasklens'][i]}") entry['ipAddressList'] = ipaddr else: entry['ipAddressList'] = [] if entry.get('ip6AddressList', None): if '_linklocal' in entry: entry['ip6AddressList'].append(entry['_linklocal']) else: entry['ip6AddressList'] = [] if entry.get('_anycastMac', ''): entry['interfaceMac'] = entry.get('macaddr', '') entry['macaddr'] = entry['_anycastMac'] if entry.get('_forwardMode', '') != 'Anycast': entry['reason'] += ', Fabric forwarding mode not enabled' entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) unnum_if_parent = entry.get('_unnum_intf', '') if unnum_if_parent: if unnum_if_parent in unnum_intf: # IPv6 has link local, so unnumbered is a v4 construct entry['ipAddressList'] = [unnum_intf[unnum_if_parent]] else: unnum_intf_entry_idx.append(entry_idx) if entry.get('_child_intf', []): unnum_intf[entry['ifname']] = [pri_ipaddr] speed = entry.get('speed', '') if speed: entry['speed'] = fix_nxos_speed(speed) entry['type'] = entry.get('type', '').lower() if entry['type'] == 'eth': entry['type'] = 'ethernet' if 'ethernet' in entry.get('type', ''): entry['type'] = 'ethernet' if entry['ifname'].startswith('Vlan'): entry['type'] = 'vlan' elif entry['ifname'].startswith('nve'): entry['type'] = 'vxlan' entry['master'] = 'bridge' elif entry['ifname'].startswith('loopback'): entry['type'] = 'loopback' if entry['type'] == 'vlan' and entry['ifname'].startswith('Vlan'): entry['vlan'] = int(entry['ifname'].split('Vlan')[1]) # have this at the end to avoid messing up processing # Fix unnumbered interface references for idx in unnum_intf_entry_idx: entry = processed_data[idx] entry['ipAddressList'] = unnum_intf.get(entry['_unnum_intf'], []) # Fix type of mgmt0 entry = entry_dict['mgmt0'] if entry: entry['type'] = 'ethernet' if drop_indices: processed_data = np.delete(processed_data, drop_indices).tolist() if new_entries: processed_data.extend(new_entries) return processed_data
def _clean_nxos_data(self, processed_data, raw_data): """Complex cleanup of NXOS interface data""" new_entries = [] unnum_intf = {} add_bridge_intf = False bridge_intf_state = "down" bridge_mtu = 1500 unnum_intf_entry_idx = [] # backtrack to interface to fix for entry_idx, entry in enumerate(processed_data): # artificial field for comparison with previous poll result entry["statusChangeTimestamp1"] = entry.get( "statusChangeTimestamp", '') if entry.get('vrf', 'default') != 'default': entry['master'] = entry['vrf'] else: entry['master'] = '' if 'routeDistinguisher' in entry: entry['macaddr'] = "00:00:00:00:00:00" entry['adminState'] = entry.get("state", "up") continue if 'reason' in entry: if entry['reason'] == '"Administratively down': entry['adminState'] = 'down' else: entry['adminState'] = 'up' portmode = entry.get('_portmode', '') if portmode == 'access' or portmode == 'trunk': entry['master'] = 'bridge' add_bridge_intf = True if entry.get('state', '') == "up": bridge_intf_state = "up" if entry.get('mtu', 1500) < bridge_mtu: bridge_mtu = entry.get('mtu', 0) portchan = entry.get('_portchannel', '') if portchan: entry['master'] = f'port-channel{portchan}' entry['type'] = 'bond_slave' if entry['ifname'].startswith('port-channel'): entry['type'] = 'bond' if 'ipAddressList' in entry: pri_ipaddr = f"{entry['ipAddressList']}/{entry['_maskLen']}" ipaddr = [pri_ipaddr] for i, elem in enumerate(entry.get('_secIPs', [])): if elem: ipaddr.append(f"{elem}/{entry['_secmasklens'][i]}") entry['ipAddressList'] = ipaddr else: entry['ipAddressList'] = [] if 'ip6AddressList' in entry: if '_linklocal' in entry: entry['ip6AddressList'].append(entry['_linklocal']) else: entry['ip6AddressList'] = [] if entry.get('_anycastMac', ''): entry['interfaceMac'] = entry['macaddr'] entry['macaddr'] = entry['_anycastMac'] if entry.get('_forwardMode', '') != 'Anycast': entry['reason'] += ', Fabric forwarding mode not enabled' entry['macaddr'] = convert_macaddr_format_to_colon( entry.get('macaddr', '0000.0000.0000')) if entry.get('_unnum_intf', ''): if entry['ifname'] in unnum_intf: # IPv6 has link local, so unnumbered is a v4 construct entry['ipAddressList'] = [unnum_intf[entry['ifname']]] else: unnum_intf_entry_idx.append(entry_idx) for elem in entry.get('_child_intf', []): unnum_intf[elem] = [pri_ipaddr] speed = entry.get('speed', '') if isinstance(speed, str) and speed.startswith("unknown enum"): entry['speed'] = 0 entry['type'] = entry.get('type', '').lower() if entry['type'] == 'eth': entry['type'] = 'ethernet' if entry['ifname'].startswith('Vlan'): entry['type'] = 'vlan' elif entry['ifname'].startswith('nve'): entry['type'] = 'vxlan' entry['master'] = 'bridge' elif entry['ifname'].startswith('loopback'): entry['type'] = 'loopback' if entry['type'] == 'vlan' and entry['ifname'].startswith('Vlan'): entry['vlan'] = int(entry['ifname'].split('Vlan')[1]) # have this at the end to avoid messing up processing # Fix unnumbered interface references for idx in unnum_intf_entry_idx: entry = processed_data[idx] entry['ipAddressList'] = unnum_intf.get(entry['ifname'], []) if new_entries: processed_data.extend(new_entries) return processed_data