def _clean_junos_data(self, processed_data, raw_data):
for entry in processed_data:
entry['bootupTimestamp'] = get_timestamp_from_junos_time(
entry['bootupTimestamp'], int(raw_data[0]["timestamp"])/1000)/1000
return self._common_data_cleaner(processed_data, raw_data)
def _clean_junos_data(self, processed_data, raw_data):
for entry in processed_data:
if entry.get('bootupTimestamp', '-') != '-':
entry['bootupTimestamp'] = (get_timestamp_from_junos_time(
entry['bootupTimestamp'],
(raw_data[0]['timestamp'] / 1000))) / 1000
return self._common_data_cleaner(processed_data, raw_data)
def _clean_junos_data(self, processed_data, raw_data):
for entry in processed_data:
vrf = entry['vrf'][0]['data']
if vrf == "master":
entry['vrf'] = "default"
else:
entry['vrf'] = vrf
entry['lastChangeTime'] = get_timestamp_from_junos_time(
entry['lastChangeTime'], raw_data[0]['timestamp']/1000)
entry['state'] = entry['state'].lower()
return processed_data
async def _parse_boottime_hostname(self, output, cb_token) -> None:
"""Parse the uptime command output"""
if output[0]["status"] == 0:
data = output[0]["data"]
bootts = re.search(r'\nSystem booted:.*\((.+) ago\)', data)
if bootts:
self.bootupTimestamp = get_timestamp_from_junos_time(
bootts.group(1), output[0]['timestamp'] / 1000)
if output[1]["status"] == 0:
data = output[0]["data"]
hmatch = re.search(r'\nHostname:\s+(\S+)\n', data)
if hmatch:
self.set_hostname(hmatch.group(1))
async def _parse_boottime_hostname(self, output, cb_token) -> None:
"""Parse the uptime command output"""
if output[0]["status"] == 0:
data = output[0]["data"]
try:
jdata = json.loads(data.replace('\n', '').strip())
if self.devtype == 'junos-qfx' or 'juniper-ex':
jdata = jdata['multi-routing-engine-results'][0]['multi-routing-engine-item'][0]
timestr = jdata['system-uptime-information'][0]['system-booted-time'][0]['time-length'][0]['attributes']
except Exception:
self.logger.warning(
f'Unable to parse junos boot time from {data}')
timestr = '{"junos:seconds": "0"}'
self.bootupTimestamp = (get_timestamp_from_junos_time(
timestr, output[0]['timestamp']/1000)/1000)
if output[1]["status"] == 0:
data = output[0]["data"]
hmatch = re.search(r'\nHostname:\s+(\S+)\n', data)
if hmatch:
self.set_hostname(hmatch.group(1))
def _clean_junos_data(self, processed_data, raw_data):
def _rename_afi_safi(afistr: str) -> str:
if afistr == "l2vpn":
afistr = "l2vpn-vpls"
elif afistr == "l2vpn-signaling":
afistr = "l2vpn-vpws"
afistr = (afistr.replace('inet-vpn', 'vpnv4').replace(
'inet6-vpn',
'vpnv6').replace('-', ' ').replace('inet6', 'ipv6').replace(
'inet',
'ipv4').replace('flow',
'flowspec').replace('evpn', 'l2vpn evpn'))
return afistr
peer_uptimes = {}
drop_indices = []
new_entries = []
for i, entry in enumerate(processed_data):
if entry['_entryType'] == 'summary':
peer_uptimes[entry['peer']] = entry['estdTime']
drop_indices.append(i)
continue
# JunOS adds entries which includes the port as IP+Port
entry['peerIP'] = entry['peerIP'].split('+')[0]
entry['peer'] = entry['peer'].split('+')[0]
entry['updateSource'] = entry['updateSource'].split('+')[0]
entry['numChanges'] = int(entry['numChanges'])
entry['updatesRx'] = int(entry['updatesRx'])
entry['updatesTx'] = int(entry['updatesTx'])
entry['asn'] = int(entry['asn'])
entry['peerAsn'] = int(entry['peerAsn'])
entry['keepaliveTime'] = int(entry['keepaliveTime'])
entry['holdTime'] = int(entry['holdTime'])
if entry['peer'] in peer_uptimes:
entry['estdTime'] = peer_uptimes[entry['peer']]
else:
entry['estdTime'] = '0d 00:00:00'
advafis = set(entry['afiSafiAdvList'].split())
rcvafis = set(entry['afiSafiRcvList'].split())
entry['afisAdvOnly'] = list(advafis.difference(rcvafis))
entry['afisRcvOnly'] = list(rcvafis.difference(advafis))
entry['afisAdvOnly'] = list(
map(_rename_afi_safi, entry['afisAdvOnly']))
entry['afisRcvOnly'] = list(
map(_rename_afi_safi, entry['afisRcvOnly']))
# Junos doesn't provide this data in neighbor, only in summary
entry['estdTime'] = get_timestamp_from_junos_time(
entry['estdTime'], raw_data[0]['timestamp'] / 1000)
if not entry.get('vrf', None):
entry['vrf'] = 'default'
if entry['state'] != 'Established':
entry['afi'] = entry['safi'] = ''
continue
# Build the mapping between pfx counts with the AFI/SAFI
# Assign counts to appropriate AFi/SAFI
table_afi_map = {}
for x in zip(entry['_tableAfiList'], entry['_tableNameList']):
table_afi_map.setdefault(x[0], []).append(x[1])
pfxrx_list = dict(zip(entry['_pfxType'], entry['_pfxRxList']))
pfxtx_list = dict(zip(entry['_pfxType'], entry['_pfxTxList']))
pfxsupp_list = dict(
zip(entry['_pfxType'], entry['_pfxSuppressList']))
pfxbest_list = dict(zip(entry['_pfxType'],
entry['_pfxBestRxList']))
for orig_elem in table_afi_map:
new_entry = deepcopy(entry)
elem = _rename_afi_safi(orig_elem)
afi, safi = elem.split()
new_entry['afi'] = afi
new_entry['safi'] = safi
new_entry['pfxRx'] = 0
new_entry['pfxTx'] = 0
new_entry['pfxBestRx'] = 0
new_entry['pfxSuppressRx'] = 0
for table in table_afi_map[orig_elem]:
new_entry['pfxRx'] += int(pfxrx_list.get(table, 0) or 0)
new_entry['pfxTx'] += int(pfxtx_list.get(table, 0) or 0)
new_entry['pfxSuppressRx'] += int(
pfxsupp_list.get(table, 0) or 0)
new_entry['pfxBestRx'] += int(
pfxbest_list.get(table, 0) or 0)
new_entry['communityTypes'] = ['standard', 'extended']
new_entry.pop('_pfxType')
new_entry.pop('_pfxRxList')
new_entry.pop('_pfxTxList')
new_entry.pop('_pfxSuppressList')
new_entry.pop('_tableAfiList')
new_entry.pop('_tableNameList')
new_entries.append(new_entry)
drop_indices.append(i)
processed_data += new_entries
processed_data = np.delete(processed_data, drop_indices).tolist()
return processed_data
def _clean_junos_data(self, processed_data, raw_data):
peer_uptimes = {}
drop_indices = []
for i, entry in enumerate(processed_data):
if entry['_entryType'] == 'summary':
peer_uptimes[entry['peer']] = entry['estdTime']
drop_indices.append(i)
continue
# JunOS adds entries which includes the port as IP+Port
entry['peerIP'] = entry['peerIP'].split('+')[0]
entry['peer'] = entry['peer'].split('+')[0]
entry['updateSource'] = entry['updateSource'].split('+')[0]
entry['numChanges'] = int(entry['numChanges'])
entry['updatesRx'] = int(entry['updatesRx'])
entry['updatesTx'] = int(entry['updatesTx'])
entry['asn'] = int(entry['asn'])
entry['peerAsn'] = int(entry['peerAsn'])
entry['keepaliveTime'] = int(entry['keepaliveTime'])
entry['holdTime'] = int(entry['holdTime'])
if entry['peer'] in peer_uptimes:
entry['estdTime'] = peer_uptimes[entry['peer']]
else:
entry['estdTime'] = '0d 00:00:00'
# Assign counts to appropriate AFi/SAFI
for i, afi in enumerate(entry.get('pfxType')):
if entry['pfxRxList'][i] is None:
entry['pfxRxList'][i] = 0
if afi == 'inet.0':
entry['v4PfxRx'] = int(entry['pfxRxList'][i])
entry['v4PfxTx'] = int(entry['pfxTxList'][i])
entry['v4Enabled'] = True
elif afi == 'inet6.0':
entry['v6PfxRx'] = int(entry['pfxRxList'][i])
entry['v6PfxTx'] = int(entry['pfxTxList'][i])
elif afi == 'bgp.evpn.0':
entry['evpnPfxRx'] = int(entry['pfxRxList'][i])
entry['evpnPfxTx'] = int(entry['pfxTxList'][i])
entry.pop('pfxType')
entry.pop('pfxRxList')
entry.pop('pfxTxList')
for afi in entry['afiSafiAdvList'].split():
if afi == 'inet-unicast':
entry['v4Advertised'] = True
elif afi == 'inet6-unicast':
entry['v6Advertised'] = True
elif afi == 'evpn':
entry['evpnAdvertised'] = True
entry.pop('afiSafiAdvList')
for afi in entry['afiSafiRcvList'].split():
if afi == 'inet-unicast':
entry['v4Received'] = True
elif afi == 'inet6-unicast':
entry['v6Received'] = True
elif afi == 'evpn':
entry['evpnReceived'] = True
entry.pop('afiSafiRcvList')
for afi in entry['afiSafiEnabledList'].split():
if afi == 'inet-unicast':
entry['v4Enabled'] = True
elif afi == 'inet6-unicast':
entry['v6Enabled'] = True
elif afi == 'evpn':
entry['evpnEnabled'] = True
entry.pop('afiSafiEnabledList')
if not entry.get('vrf', None):
entry['vrf'] = 'default'
# Junos doesn't provide this data in neighbor, only in summary
entry['estdTime'] = get_timestamp_from_junos_time(
entry['estdTime'], raw_data[0]['timestamp'] / 1000)
processed_data = np.delete(processed_data, drop_indices).tolist()
return processed_data
def _clean_junos_data(self, processed_data, raw_data):
"""Cleanup IP addresses and such"""
entry_dict = defaultdict(dict)
drop_indices = []
new_entries = []
for i, entry in enumerate(processed_data):
if entry['type'] == 'vrf':
self._assign_vrf(entry, entry_dict)
if entry['ifname'] == 'default':
drop_indices.append(i)
continue
if entry['type'] is None:
continue
if entry['type'] == 'virtual-switch':
# TODO: Handle this properly
continue
if not entry_dict[entry['ifname']]:
entry_dict[entry['ifname']] = entry
if entry.get('mtu', 0) == 'Unlimited':
entry['mtu'] = 65536
entry['mtu'] = int(entry.get('mtu', 0))
if entry['type']:
entry['type'] = entry['type'].lower()
if 'statusChangeTimestamp' in entry:
ts = get_timestamp_from_junos_time(
entry['statusChangeTimestamp'], raw_data[0]['timestamp']/1000)
entry['statusChangeTimestamp'] = int(ts)
if entry.get('speed', ''):
if entry['speed'].endswith('mbps'):
entry['speed'] = int(entry['speed'].split('mb')[0])
elif entry['speed'].endswith('Gbps'):
entry['speed'] = int(entry['speed'].split('Gb')[0])*1000
elif entry['speed'] == 'Unlimited':
entry['speed'] = 0
if entry['master'] == 'Ethernet-Bridge':
entry['master'] = 'bridge'
elif entry['master'] == 'unknown':
if entry['ifname'].startswith('jsv'):
entry['type'] = 'sflowMonitor'
else:
entry['type'] = 'virtual'
entry['master'] = ''
else:
entry['master'] = ''
if entry['type'] == 'vxlan-tunnel-endpoint':
entry['type'] = 'vtep'
elif entry['type'] == 'interface-specific':
entry['type'] = 'tap'
# Process the logical interfaces which are too deep to be parsed
# efficiently by the parser right now
if entry['logicalIfname'] == [] or entry['afi'] == [None]:
entry.pop('logicalIfname')
entry.pop('afi')
entry.pop('vlanName')
continue
# Uninteresting logical interface
gwmacs = entry.get('_gwMacaddr', [])
for i, ifname in enumerate(entry['logicalIfname']):
v4addresses = []
v6addresses = []
macaddr = None
if len(gwmacs) > i:
macaddr = gwmacs[i]
if not macaddr:
macaddr = entry.get('macaddr', '')
if entry['afi'][i] is None:
continue
for x in entry['afi'][i]:
if isinstance(x, list):
afi = x[0].get('interface-address', None)
else:
afi = x.get('interface-address', None)
if afi and afi is not None:
break
foo = x.get('address-family-name', None)
if foo and (foo[0].get('data', None) == "eth-switch"):
afi = []
break
else:
continue
if entry_dict[ifname]:
vrf = entry_dict[ifname]['vrf']
else:
vrf = ''
new_entry = {'ifname': ifname,
'mtu': entry['afi'][i][0].get(
'mtu', [{'data': 0}])[0]['data'],
'type': 'logical',
'speed': entry['speed'],
'master': vrf,
'macaddr': macaddr,
'adminState': 'up',
'description': entry['description'],
'statusChangeTimestamp':
entry['statusChangeTimestamp'],
}
new_entries.append(new_entry)
entry_dict[new_entry['ifname']] = new_entry
if (entry['logicalIfflags'][i][0].get('iff-up') or
entry.get('type') == 'loopback'):
new_entry['state'] = 'up'
else:
new_entry['state'] = 'down'
if new_entry['mtu'] == 'Unlimited':
new_entry['mtu'] = 65536
else:
new_entry['mtu'] = int(new_entry['mtu'])
for x in afi:
address = (x.get("ifa-local")[0]["data"] + '/' +
x.get("ifa-destination", [{"data": "0/32"}])[0]
["data"].split("/")[1])
if ':' in address:
v6addresses.append(address)
else:
v4addresses.append(address)
vlanName = entry['vlanName'][i]
if vlanName is not None:
new_entry['vlanName'] = vlanName
else:
new_entry['vlanName'] = '-'
new_entry['ip6AddressList'] = v6addresses
new_entry['ipAddressList'] = v4addresses
entry.pop('vlanName')
if drop_indices:
processed_data = np.delete(processed_data, drop_indices).tolist()
if new_entries:
processed_data.extend(new_entries)
return processed_data
def _clean_junos_data(self, processed_data, raw_data):
"""Cleanup IP addresses and such"""
new_entries = [] # Add new interface entries for logical ifs
for entry in processed_data:
if entry['mtu'] == 'Unlimited':
entry['mtu'] = 65536
else:
entry['mtu'] = int(entry['mtu'])
if entry['type']:
entry['type'] = entry['type'].lower()
if (entry['statusChangeTimestamp'] == 'Never'
or entry['statusChangeTimestamp'] is None):
entry['statusChangeTimestamp'] = 0
# artificial field for comparison with previous poll result
entry["statusChangeTimestamp1"] = 0
else:
ts_str = re.match(r'.*\((.+) ago\)$',
entry['statusChangeTimestamp'])
ts1_str = re.match(r'^[^(]*', entry['statusChangeTimestamp'])
ts = get_timestamp_from_junos_time(
ts_str.group(1), raw_data[0]['timestamp'] / 1000)
entry['statusChangeTimestamp'] = int(ts)
# artificial field for comparison with previous poll result
entry["statusChangeTimestamp1"] = ts1_str
if entry['speed']:
if entry['speed'].endswith('mbps'):
entry['speed'] = int(entry['speed'].split('mb')[0])
elif entry['speed'].endswith('Gbps'):
entry['speed'] = int(entry['speed'].split('Gb')[0]) * 1000
elif entry['speed'] == 'Unlimited':
entry['speed'] = 0
entry['ifname'] = entry['ifname'].replace('/', '-')
if entry['master'] == 'Ethernet-Bridge':
entry['master'] = 'bridge'
elif entry['master'] == 'unknown':
if entry['ifname'].startswith('jsv'):
entry['type'] = 'sflowMonitor'
else:
entry['type'] = 'virtual'
entry['master'] = None
else:
entry['master'] = None
if entry['type'] == 'vxlan-tunnel-endpoint':
entry['type'] = 'vtep'
elif entry['type'] == 'interface-specific':
entry['type'] = 'tap'
# Process the logical interfaces which are too deep to be parsed
# efficiently by the parser right now
if entry['logicalIfname'] == [] or entry['afi'] == [None]:
entry.pop('logicalIfname')
entry.pop('afi')
entry.pop('vlanName')
continue
# Uninteresting logical interface
for i, ifname in enumerate(entry['logicalIfname']):
v4addresses = []
v6addresses = []
if entry['afi'][i] is None:
continue
for x in entry['afi'][i]:
if isinstance(x, list):
afi = x[0].get('interface-address', None)
else:
afi = x.get('interface-address', None)
if afi and afi is not None:
break
foo = x.get('address-family-name', None)
if foo and (foo[0].get('data', None) == "eth-switch"):
afi = []
break
else:
continue
new_entry = {
'ifname': ifname,
'origIfname': ifname,
'mtu': entry['afi'][i][0].get('mtu', [{
'data': 0
}])[0]['data'],
'type': 'logical',
'speed': entry['speed'],
'master': '',
'description': entry['description'],
'statusChangeTimestamp': entry['statusChangeTimestamp'],
}
if entry['logicalIfflags'][i][0].get('iff-up'):
new_entry['state'] = 'up'
else:
new_entry['state'] = 'down'
if new_entry['mtu'] == 'Unlimited':
new_entry['mtu'] = 65536
else:
new_entry['mtu'] = int(new_entry['mtu'])
for x in afi:
address = (x.get("ifa-local")[0]["data"] + '/' +
x.get("ifa-destination", [{
"data": "0/32"
}])[0]["data"].split("/")[1])
if ':' in address:
v6addresses.append(address)
else:
v4addresses.append(address)
vlanName = entry['vlanName'][i]
if vlanName is not None:
new_entry['vlanName'] = vlanName
else:
new_entry['vlanName'] = '-'
new_entry['ip6AddressList'] = v6addresses
new_entry['ipAddressList'] = v4addresses
new_entry['ifname'] = new_entry['ifname'].replace('/', '-')
new_entries.append(new_entry)
entry.pop('vlanName')
if new_entries:
processed_data.extend(new_entries)
return processed_data
