def _clean_junos_data(self, processed_data, raw_data): for entry in processed_data: entry['bootupTimestamp'] = get_timestamp_from_junos_time( entry['bootupTimestamp'], int(raw_data[0]["timestamp"])/1000)/1000 return self._common_data_cleaner(processed_data, raw_data)
def _clean_junos_data(self, processed_data, raw_data): for entry in processed_data: if entry.get('bootupTimestamp', '-') != '-': entry['bootupTimestamp'] = (get_timestamp_from_junos_time( entry['bootupTimestamp'], (raw_data[0]['timestamp'] / 1000))) / 1000 return self._common_data_cleaner(processed_data, raw_data)
def _clean_junos_data(self, processed_data, raw_data): for entry in processed_data: vrf = entry['vrf'][0]['data'] if vrf == "master": entry['vrf'] = "default" else: entry['vrf'] = vrf entry['lastChangeTime'] = get_timestamp_from_junos_time( entry['lastChangeTime'], raw_data[0]['timestamp']/1000) entry['state'] = entry['state'].lower() return processed_data
async def _parse_boottime_hostname(self, output, cb_token) -> None: """Parse the uptime command output""" if output[0]["status"] == 0: data = output[0]["data"] bootts = re.search(r'\nSystem booted:.*\((.+) ago\)', data) if bootts: self.bootupTimestamp = get_timestamp_from_junos_time( bootts.group(1), output[0]['timestamp'] / 1000) if output[1]["status"] == 0: data = output[0]["data"] hmatch = re.search(r'\nHostname:\s+(\S+)\n', data) if hmatch: self.set_hostname(hmatch.group(1))
async def _parse_boottime_hostname(self, output, cb_token) -> None: """Parse the uptime command output""" if output[0]["status"] == 0: data = output[0]["data"] try: jdata = json.loads(data.replace('\n', '').strip()) if self.devtype == 'junos-qfx' or 'juniper-ex': jdata = jdata['multi-routing-engine-results'][0]['multi-routing-engine-item'][0] timestr = jdata['system-uptime-information'][0]['system-booted-time'][0]['time-length'][0]['attributes'] except Exception: self.logger.warning( f'Unable to parse junos boot time from {data}') timestr = '{"junos:seconds": "0"}' self.bootupTimestamp = (get_timestamp_from_junos_time( timestr, output[0]['timestamp']/1000)/1000) if output[1]["status"] == 0: data = output[0]["data"] hmatch = re.search(r'\nHostname:\s+(\S+)\n', data) if hmatch: self.set_hostname(hmatch.group(1))
def _clean_junos_data(self, processed_data, raw_data): def _rename_afi_safi(afistr: str) -> str: if afistr == "l2vpn": afistr = "l2vpn-vpls" elif afistr == "l2vpn-signaling": afistr = "l2vpn-vpws" afistr = (afistr.replace('inet-vpn', 'vpnv4').replace( 'inet6-vpn', 'vpnv6').replace('-', ' ').replace('inet6', 'ipv6').replace( 'inet', 'ipv4').replace('flow', 'flowspec').replace('evpn', 'l2vpn evpn')) return afistr peer_uptimes = {} drop_indices = [] new_entries = [] for i, entry in enumerate(processed_data): if entry['_entryType'] == 'summary': peer_uptimes[entry['peer']] = entry['estdTime'] drop_indices.append(i) continue # JunOS adds entries which includes the port as IP+Port entry['peerIP'] = entry['peerIP'].split('+')[0] entry['peer'] = entry['peer'].split('+')[0] entry['updateSource'] = entry['updateSource'].split('+')[0] entry['numChanges'] = int(entry['numChanges']) entry['updatesRx'] = int(entry['updatesRx']) entry['updatesTx'] = int(entry['updatesTx']) entry['asn'] = int(entry['asn']) entry['peerAsn'] = int(entry['peerAsn']) entry['keepaliveTime'] = int(entry['keepaliveTime']) entry['holdTime'] = int(entry['holdTime']) if entry['peer'] in peer_uptimes: entry['estdTime'] = peer_uptimes[entry['peer']] else: entry['estdTime'] = '0d 00:00:00' advafis = set(entry['afiSafiAdvList'].split()) rcvafis = set(entry['afiSafiRcvList'].split()) entry['afisAdvOnly'] = list(advafis.difference(rcvafis)) entry['afisRcvOnly'] = list(rcvafis.difference(advafis)) entry['afisAdvOnly'] = list( map(_rename_afi_safi, entry['afisAdvOnly'])) entry['afisRcvOnly'] = list( map(_rename_afi_safi, entry['afisRcvOnly'])) # Junos doesn't provide this data in neighbor, only in summary entry['estdTime'] = get_timestamp_from_junos_time( entry['estdTime'], raw_data[0]['timestamp'] / 1000) if not entry.get('vrf', None): entry['vrf'] = 'default' if entry['state'] != 'Established': entry['afi'] = entry['safi'] = '' continue # Build the mapping between pfx counts with the AFI/SAFI # Assign counts to appropriate AFi/SAFI table_afi_map = {} for x in zip(entry['_tableAfiList'], entry['_tableNameList']): table_afi_map.setdefault(x[0], []).append(x[1]) pfxrx_list = dict(zip(entry['_pfxType'], entry['_pfxRxList'])) pfxtx_list = dict(zip(entry['_pfxType'], entry['_pfxTxList'])) pfxsupp_list = dict( zip(entry['_pfxType'], entry['_pfxSuppressList'])) pfxbest_list = dict(zip(entry['_pfxType'], entry['_pfxBestRxList'])) for orig_elem in table_afi_map: new_entry = deepcopy(entry) elem = _rename_afi_safi(orig_elem) afi, safi = elem.split() new_entry['afi'] = afi new_entry['safi'] = safi new_entry['pfxRx'] = 0 new_entry['pfxTx'] = 0 new_entry['pfxBestRx'] = 0 new_entry['pfxSuppressRx'] = 0 for table in table_afi_map[orig_elem]: new_entry['pfxRx'] += int(pfxrx_list.get(table, 0) or 0) new_entry['pfxTx'] += int(pfxtx_list.get(table, 0) or 0) new_entry['pfxSuppressRx'] += int( pfxsupp_list.get(table, 0) or 0) new_entry['pfxBestRx'] += int( pfxbest_list.get(table, 0) or 0) new_entry['communityTypes'] = ['standard', 'extended'] new_entry.pop('_pfxType') new_entry.pop('_pfxRxList') new_entry.pop('_pfxTxList') new_entry.pop('_pfxSuppressList') new_entry.pop('_tableAfiList') new_entry.pop('_tableNameList') new_entries.append(new_entry) drop_indices.append(i) processed_data += new_entries processed_data = np.delete(processed_data, drop_indices).tolist() return processed_data
def _clean_junos_data(self, processed_data, raw_data): peer_uptimes = {} drop_indices = [] for i, entry in enumerate(processed_data): if entry['_entryType'] == 'summary': peer_uptimes[entry['peer']] = entry['estdTime'] drop_indices.append(i) continue # JunOS adds entries which includes the port as IP+Port entry['peerIP'] = entry['peerIP'].split('+')[0] entry['peer'] = entry['peer'].split('+')[0] entry['updateSource'] = entry['updateSource'].split('+')[0] entry['numChanges'] = int(entry['numChanges']) entry['updatesRx'] = int(entry['updatesRx']) entry['updatesTx'] = int(entry['updatesTx']) entry['asn'] = int(entry['asn']) entry['peerAsn'] = int(entry['peerAsn']) entry['keepaliveTime'] = int(entry['keepaliveTime']) entry['holdTime'] = int(entry['holdTime']) if entry['peer'] in peer_uptimes: entry['estdTime'] = peer_uptimes[entry['peer']] else: entry['estdTime'] = '0d 00:00:00' # Assign counts to appropriate AFi/SAFI for i, afi in enumerate(entry.get('pfxType')): if entry['pfxRxList'][i] is None: entry['pfxRxList'][i] = 0 if afi == 'inet.0': entry['v4PfxRx'] = int(entry['pfxRxList'][i]) entry['v4PfxTx'] = int(entry['pfxTxList'][i]) entry['v4Enabled'] = True elif afi == 'inet6.0': entry['v6PfxRx'] = int(entry['pfxRxList'][i]) entry['v6PfxTx'] = int(entry['pfxTxList'][i]) elif afi == 'bgp.evpn.0': entry['evpnPfxRx'] = int(entry['pfxRxList'][i]) entry['evpnPfxTx'] = int(entry['pfxTxList'][i]) entry.pop('pfxType') entry.pop('pfxRxList') entry.pop('pfxTxList') for afi in entry['afiSafiAdvList'].split(): if afi == 'inet-unicast': entry['v4Advertised'] = True elif afi == 'inet6-unicast': entry['v6Advertised'] = True elif afi == 'evpn': entry['evpnAdvertised'] = True entry.pop('afiSafiAdvList') for afi in entry['afiSafiRcvList'].split(): if afi == 'inet-unicast': entry['v4Received'] = True elif afi == 'inet6-unicast': entry['v6Received'] = True elif afi == 'evpn': entry['evpnReceived'] = True entry.pop('afiSafiRcvList') for afi in entry['afiSafiEnabledList'].split(): if afi == 'inet-unicast': entry['v4Enabled'] = True elif afi == 'inet6-unicast': entry['v6Enabled'] = True elif afi == 'evpn': entry['evpnEnabled'] = True entry.pop('afiSafiEnabledList') if not entry.get('vrf', None): entry['vrf'] = 'default' # Junos doesn't provide this data in neighbor, only in summary entry['estdTime'] = get_timestamp_from_junos_time( entry['estdTime'], raw_data[0]['timestamp'] / 1000) processed_data = np.delete(processed_data, drop_indices).tolist() return processed_data
def _clean_junos_data(self, processed_data, raw_data): """Cleanup IP addresses and such""" entry_dict = defaultdict(dict) drop_indices = [] new_entries = [] for i, entry in enumerate(processed_data): if entry['type'] == 'vrf': self._assign_vrf(entry, entry_dict) if entry['ifname'] == 'default': drop_indices.append(i) continue if entry['type'] is None: continue if entry['type'] == 'virtual-switch': # TODO: Handle this properly continue if not entry_dict[entry['ifname']]: entry_dict[entry['ifname']] = entry if entry.get('mtu', 0) == 'Unlimited': entry['mtu'] = 65536 entry['mtu'] = int(entry.get('mtu', 0)) if entry['type']: entry['type'] = entry['type'].lower() if 'statusChangeTimestamp' in entry: ts = get_timestamp_from_junos_time( entry['statusChangeTimestamp'], raw_data[0]['timestamp']/1000) entry['statusChangeTimestamp'] = int(ts) if entry.get('speed', ''): if entry['speed'].endswith('mbps'): entry['speed'] = int(entry['speed'].split('mb')[0]) elif entry['speed'].endswith('Gbps'): entry['speed'] = int(entry['speed'].split('Gb')[0])*1000 elif entry['speed'] == 'Unlimited': entry['speed'] = 0 if entry['master'] == 'Ethernet-Bridge': entry['master'] = 'bridge' elif entry['master'] == 'unknown': if entry['ifname'].startswith('jsv'): entry['type'] = 'sflowMonitor' else: entry['type'] = 'virtual' entry['master'] = '' else: entry['master'] = '' if entry['type'] == 'vxlan-tunnel-endpoint': entry['type'] = 'vtep' elif entry['type'] == 'interface-specific': entry['type'] = 'tap' # Process the logical interfaces which are too deep to be parsed # efficiently by the parser right now if entry['logicalIfname'] == [] or entry['afi'] == [None]: entry.pop('logicalIfname') entry.pop('afi') entry.pop('vlanName') continue # Uninteresting logical interface gwmacs = entry.get('_gwMacaddr', []) for i, ifname in enumerate(entry['logicalIfname']): v4addresses = [] v6addresses = [] macaddr = None if len(gwmacs) > i: macaddr = gwmacs[i] if not macaddr: macaddr = entry.get('macaddr', '') if entry['afi'][i] is None: continue for x in entry['afi'][i]: if isinstance(x, list): afi = x[0].get('interface-address', None) else: afi = x.get('interface-address', None) if afi and afi is not None: break foo = x.get('address-family-name', None) if foo and (foo[0].get('data', None) == "eth-switch"): afi = [] break else: continue if entry_dict[ifname]: vrf = entry_dict[ifname]['vrf'] else: vrf = '' new_entry = {'ifname': ifname, 'mtu': entry['afi'][i][0].get( 'mtu', [{'data': 0}])[0]['data'], 'type': 'logical', 'speed': entry['speed'], 'master': vrf, 'macaddr': macaddr, 'adminState': 'up', 'description': entry['description'], 'statusChangeTimestamp': entry['statusChangeTimestamp'], } new_entries.append(new_entry) entry_dict[new_entry['ifname']] = new_entry if (entry['logicalIfflags'][i][0].get('iff-up') or entry.get('type') == 'loopback'): new_entry['state'] = 'up' else: new_entry['state'] = 'down' if new_entry['mtu'] == 'Unlimited': new_entry['mtu'] = 65536 else: new_entry['mtu'] = int(new_entry['mtu']) for x in afi: address = (x.get("ifa-local")[0]["data"] + '/' + x.get("ifa-destination", [{"data": "0/32"}])[0] ["data"].split("/")[1]) if ':' in address: v6addresses.append(address) else: v4addresses.append(address) vlanName = entry['vlanName'][i] if vlanName is not None: new_entry['vlanName'] = vlanName else: new_entry['vlanName'] = '-' new_entry['ip6AddressList'] = v6addresses new_entry['ipAddressList'] = v4addresses entry.pop('vlanName') if drop_indices: processed_data = np.delete(processed_data, drop_indices).tolist() if new_entries: processed_data.extend(new_entries) return processed_data
def _clean_junos_data(self, processed_data, raw_data): """Cleanup IP addresses and such""" new_entries = [] # Add new interface entries for logical ifs for entry in processed_data: if entry['mtu'] == 'Unlimited': entry['mtu'] = 65536 else: entry['mtu'] = int(entry['mtu']) if entry['type']: entry['type'] = entry['type'].lower() if (entry['statusChangeTimestamp'] == 'Never' or entry['statusChangeTimestamp'] is None): entry['statusChangeTimestamp'] = 0 # artificial field for comparison with previous poll result entry["statusChangeTimestamp1"] = 0 else: ts_str = re.match(r'.*\((.+) ago\)$', entry['statusChangeTimestamp']) ts1_str = re.match(r'^[^(]*', entry['statusChangeTimestamp']) ts = get_timestamp_from_junos_time( ts_str.group(1), raw_data[0]['timestamp'] / 1000) entry['statusChangeTimestamp'] = int(ts) # artificial field for comparison with previous poll result entry["statusChangeTimestamp1"] = ts1_str if entry['speed']: if entry['speed'].endswith('mbps'): entry['speed'] = int(entry['speed'].split('mb')[0]) elif entry['speed'].endswith('Gbps'): entry['speed'] = int(entry['speed'].split('Gb')[0]) * 1000 elif entry['speed'] == 'Unlimited': entry['speed'] = 0 entry['ifname'] = entry['ifname'].replace('/', '-') if entry['master'] == 'Ethernet-Bridge': entry['master'] = 'bridge' elif entry['master'] == 'unknown': if entry['ifname'].startswith('jsv'): entry['type'] = 'sflowMonitor' else: entry['type'] = 'virtual' entry['master'] = None else: entry['master'] = None if entry['type'] == 'vxlan-tunnel-endpoint': entry['type'] = 'vtep' elif entry['type'] == 'interface-specific': entry['type'] = 'tap' # Process the logical interfaces which are too deep to be parsed # efficiently by the parser right now if entry['logicalIfname'] == [] or entry['afi'] == [None]: entry.pop('logicalIfname') entry.pop('afi') entry.pop('vlanName') continue # Uninteresting logical interface for i, ifname in enumerate(entry['logicalIfname']): v4addresses = [] v6addresses = [] if entry['afi'][i] is None: continue for x in entry['afi'][i]: if isinstance(x, list): afi = x[0].get('interface-address', None) else: afi = x.get('interface-address', None) if afi and afi is not None: break foo = x.get('address-family-name', None) if foo and (foo[0].get('data', None) == "eth-switch"): afi = [] break else: continue new_entry = { 'ifname': ifname, 'origIfname': ifname, 'mtu': entry['afi'][i][0].get('mtu', [{ 'data': 0 }])[0]['data'], 'type': 'logical', 'speed': entry['speed'], 'master': '', 'description': entry['description'], 'statusChangeTimestamp': entry['statusChangeTimestamp'], } if entry['logicalIfflags'][i][0].get('iff-up'): new_entry['state'] = 'up' else: new_entry['state'] = 'down' if new_entry['mtu'] == 'Unlimited': new_entry['mtu'] = 65536 else: new_entry['mtu'] = int(new_entry['mtu']) for x in afi: address = (x.get("ifa-local")[0]["data"] + '/' + x.get("ifa-destination", [{ "data": "0/32" }])[0]["data"].split("/")[1]) if ':' in address: v6addresses.append(address) else: v4addresses.append(address) vlanName = entry['vlanName'][i] if vlanName is not None: new_entry['vlanName'] = vlanName else: new_entry['vlanName'] = '-' new_entry['ip6AddressList'] = v6addresses new_entry['ipAddressList'] = v4addresses new_entry['ifname'] = new_entry['ifname'].replace('/', '-') new_entries.append(new_entry) entry.pop('vlanName') if new_entries: processed_data.extend(new_entries) return processed_data