def get(self, **kwargs) -> pd.DataFrame:
"""Class-specific to extract info from addnl tables"""
drop_cols = []
addnl_fields = kwargs.pop('addnl_fields', [])
columns = kwargs.get('columns', [])
if 'ifname' not in columns and 'ifname' not in addnl_fields:
addnl_fields.append('ifname')
drop_cols.append('ifname')
if (columns == ['default'] or columns == ['*'] or
'remoteVtepCnt' in columns):
getVtepCnt = True
if columns != ['*'] and 'remoteVtepList' not in columns:
addnl_fields.append('remoteVtepList')
drop_cols.append('remoteVtepList')
else:
getVtepCnt = False
df = super().get(addnl_fields=addnl_fields, **kwargs)
if df.empty:
return df
if getVtepCnt:
df.insert(len(df.columns)-4, 'remoteVtepCnt',
df.remoteVtepList.str.len())
# See if we can retrieve the info to fill out the rest of the data
# Start with VLAN values
if 'vlan' not in df.columns:
return df.drop(columns=drop_cols, errors='ignore')
iflist = df[df.vlan == 0]['ifname'].to_list()
if iflist:
ifdf = IfObj(context=self.ctxt).get(
namespace=kwargs.get('namespace'), ifname=iflist,
columns=['namespace', 'hostname', 'ifname', 'state', 'vlan',
'vni'])
if not ifdf.empty:
df = df.merge(ifdf, on=['namespace', 'hostname', 'ifname',
'vni'], how='left',
suffixes=('', '_y')) \
.drop(columns=['timestamp_y', 'state_y'])
df['vlan'] = np.where(df['vlan_y'], df['vlan_y'], df['vlan'])
df.drop(columns=['vlan_y'], inplace=True)
# Fill out the numMacs and numArps columns if we can
if 'numMacs' in df.columns:
vlanlist = list(set(df[df.numMacs == 0]['vlan'].to_list()))
else:
vlanlist = []
if vlanlist:
if 'numMacs' in df.columns:
macdf = MacsObj(context=self.ctxt).get(
namespace=kwargs.get('namespace'))
df['numMacs'] = df.apply(self._count_macs, axis=1,
args=(macdf, ))
return df.drop(columns=drop_cols, errors='ignore')
def aver(self, **kwargs) -> pd.DataFrame:
"""Assert for EVPN Data"""
assert_cols = [
"namespace", "hostname", "vni", "remoteVtepList", "vrf",
"mcastGroup", "type", "priVtepIp", "state", "l2VniList", "ifname",
"secVtepIp"
]
kwargs.pop("columns", None) # Loose whatever's passed
status = kwargs.pop('status', 'all')
df = self.get(columns=assert_cols, **kwargs)
if df.empty:
df = pd.DataFrame(columns=assert_cols)
if status != 'pass':
df['assertReason'] = 'No data found'
df['assert'] = 'fail'
return df
df["assertReason"] = [[] for _ in range(len(df))]
# Gather the unique set of VTEPs per VNI
vteps_df = df.explode(column='remoteVtepList') \
.dropna(how='any') \
.groupby(by=['vni', 'type'])['remoteVtepList'] \
.aggregate(lambda x: x.unique().tolist()) \
.reset_index() \
.dropna(how='any') \
.rename(columns={'remoteVtepList': 'allVteps'})
if not vteps_df.empty:
her_df = df.merge(vteps_df)
else:
her_df = pd.DataFrame()
if (not her_df.empty and (her_df.remoteVtepList.str.len() != 0).any()):
# Check if every VTEP we know is reachable
rdf = RoutesObj(context=self.ctxt).get(
namespace=kwargs.get('namespace'), vrf='default')
if not rdf.empty:
rdf['prefixlen'] = rdf['prefix'].str.split('/') \
.str[1].astype('int')
self._routes_df = rdf
her_df["assertReason"] += her_df.apply(self._is_vtep_reachable,
axis=1)
mcast_df = df.query('mcastGroup != "0.0.0.0"')
if not mcast_df.empty:
# Ensure that all VNIs have at most one multicast group associated
# per namespace
mismatched_vni_df = mcast_df \
.groupby(by=['namespace', 'vni'])['mcastGroup'] \
.unique() \
.reset_index() \
.dropna() \
.query('mcastGroup.str.len() != 1')
if not mismatched_vni_df.empty:
df['assertReason'] += df.apply(
lambda x, err_df: ['VNI has multiple mcast group']
if (x['namespace'] in err_df['namespace'] and x['vni'] in
err_df['vni']) else [],
axis=1,
args=(mismatched_vni_df, ))
elif not her_df.empty:
# Every VTEP has info about every other VTEP for a given VNI
her_df["assertReason"] += her_df.apply(self._all_vteps_present,
axis=1)
# State is up
df["assertReason"] += df.apply(
lambda x: ['interface is down']
if x['type'] == "L2" and x['state'] != "up" else [],
axis=1)
devices = df["hostname"].unique().tolist()
ifdf = IfObj(context=self.ctxt) \
.get(namespace=kwargs.get("namespace", ""), hostname=devices,
type='vxlan')
df = df.merge(
ifdf[['namespace', 'hostname', 'ifname', 'master', 'vlan']],
on=['namespace', 'hostname', 'ifname'],
how='left')
# vxlan interfaces, if defined, for every VNI is part of bridge
# We ensure this is true artificially for NXOS, ignored for JunOS.
df["assertReason"] += df.apply(
lambda x: ['vni not in bridge'] if (x['type'] == "L2" and x[
'ifname'] != '-' and x['master'] != "bridge") else [],
axis=1)
# mac_df = MacsObj(context=self.ctxt) \
# .get(namespace=kwargs.get("namespace", ""),
# macaddr=["00:00:00:00:00:00"])
# # Assert that we have HER for every remote VTEP
# df['assertReason'] += df.apply(self._is_her_good,
# args=(mac_df, ), axis=1)
# Fill out the assert column
df['assert'] = df.apply(lambda x: 'pass'
if not len(x.assertReason) else 'fail',
axis=1)
if status == 'fail':
df = df.query('assertReason.str.len() != 0')
elif status == "pass":
df = df.query('assertReason.str.len() == 0')
return df[['namespace', 'hostname', 'vni', 'type',
'assertReason', 'assert', 'timestamp']] \
.explode(column='assertReason') \
.fillna({'assertReason': '-'})
def aver(self, **kwargs) -> pd.DataFrame:
"""BGP Assert"""
assert_cols = [
"namespace", "hostname", "vrf", "peer", "asn", "state", "peerAsn",
"v4Enabled", "v6Enabled", "evpnEnabled", "v4Advertised",
"v6Advertised", "evpnAdvertised", "v4Received", "v6Received",
"evpnReceived", "bfdStatus", "reason", "notifcnReason"
]
kwargs.pop("columns", None) # Loose whatever's passed
df = self.get(columns=assert_cols, **kwargs)
if df.empty:
return pd.DataFrame()
if_df = IfObj(context=self.ctxt).get(namespace=kwargs.get("namespace", ""),
columns=['namespace', 'hostname',
'ifname', 'state']) \
.rename(columns={'state': 'ifState'})
lldp_df = LldpObj(context=self.ctxt).get(
namespace=kwargs.get("namespace", ""),
columns=[
'namespace', 'hostname', 'ifname', 'peerHostname', 'peerIfname'
])
# Get the dataframes we need for processing
df['cif'] = df.apply(self._get_connect_if, axis=1)
df['ifname'] = df['cif'].str.split('.').str[0]
df = df.merge(if_df, left_on=['namespace', 'hostname', 'cif'],
right_on=['namespace', 'hostname', 'ifname'],
how='left') \
.drop(columns=['timestamp_y', 'ifname_y']) \
.rename(columns={'ifname_x': 'ifname'})
# We split off at this point to avoid merging mess because of lack of
# LLDP info
df = df.merge(lldp_df, on=['namespace', 'hostname', 'ifname'], how='left') \
.drop(columns=['timestamp']) \
.rename(columns={'timestamp_x': 'timestamp'})
# Some munging to handle subinterfaces
df['xx'] = df['peerIfname'] + '.' + df['cif'].str.split('.').str[1]
df['peerIfname'] = df['xx'].where(df['xx'].notnull(), df['peerIfname'])
df.drop(columns=['xx'], inplace=True)
df = df.merge(df, left_on=['namespace', 'hostname', 'cif'],
right_on=['namespace', 'peerHostname',
'peerIfname'], how='left') \
.drop(columns=['peerIfname_y', 'timestamp_y', 'cif_y',
'ifState_y', 'reason_y', 'notifcnReason_y']) \
.rename(columns={'timestamp_x': 'timestamp',
'cif_x': 'cif', 'ifState_x': 'ifState',
'reason_x': 'reason',
'notifcnReason_x': 'notifcnReason'})
df['peer_y'] = df['peer_y'].astype(str) \
.where(df['peer_y'].notnull(), '')
df["assertReason"] = [[] for _ in range(len(df))]
# Now all sessions with NaN in the oif column have no connected route
df['assertReason'] += df.apply(
lambda x: ["outgoing link down"]
if x['cif'] and x['ifState'] != "up" else [],
axis=1)
df['assertReason'] += df.apply(lambda x: ["no route to peer"]
if not len(x['cif']) else [],
axis=1)
df['assertReason'] += df.apply(self._check_afi_safi, axis=1)
df['assertReason'] += df.apply(
lambda x: ["asn mismatch"]
if x['peer_y'] and ((x["asn_x"] != x["peerAsn_y"]) or
(x['asn_y'] != x['peerAsn_x'])) else [],
axis=1)
df['assertReason'] += df.apply(
lambda x: [f"{x['reason']}:{x['notifcnReason']}"]
if (x['reason'] and x['state_x'] != 'Established') else [],
axis=1)
df['assert'] = df.apply(lambda x: 'pass'
if not len(x.assertReason) else 'fail',
axis=1)
return (df[[
'namespace', 'hostname_x', 'vrf_x', 'peer_x', 'asn_x', 'peerAsn_x',
'state_x', 'peerHostname_x', 'vrf_y', 'peer_y', 'asn_y',
'peerAsn_y', 'assert', 'assertReason', 'timestamp'
]].rename(columns={
'hostname_x': 'hostname',
'vrf_x': 'vrf',
'peer_x': 'peer',
'asn_x': 'asn',
'peerAsn_x': 'peerAsn',
'state_x': 'state',
'peerHostname_x': 'hostnamePeer',
'vrf_y': 'vrfPeer',
'peer_y': 'peerPeer',
'asn_y': 'asnPeer',
'peerAsn_y': 'peerAsnPeer'
},
copy=False).explode(column="assertReason").astype(
{
'asn': 'Int64',
'peerAsn': 'Int64',
'asnPeer': 'Int64',
'peerAsnPeer': 'Int64'
},
copy=False).fillna({'assertReason': '-'}))
def aver(self, **kwargs) -> pd.DataFrame:
"""Assert for EVPN Data"""
assert_cols = [
"namespace", "hostname", "vni", "remoteVtepList", "vrf", "type",
"srcVtepIp", "state", "l2VniList", "ifname"
]
kwargs.pop("columns", None) # Loose whatever's passed
df = self.get(columns=assert_cols, **kwargs)
if df.empty:
return pd.DataFrame(columns=assert_cols)
# Gather the unique set of VTEPs per VNI
vteps_df = df.explode(column='remoteVtepList') \
.dropna(how='any') \
.groupby(by=['vni', 'type'])['remoteVtepList'] \
.aggregate(lambda x: x.unique().tolist()) \
.reset_index() \
.dropna(how='any') \
.rename(columns={'remoteVtepList': 'allVteps'})
df = df.merge(vteps_df)
df["assertReason"] = [[] for _ in range(len(df))]
# Every VTEP has info about every other VTEP for a given VNI
df["assertReason"] += df.apply(self._all_vteps_present, axis=1)
# Every VTEP is reachable
df["assertReason"] += df.apply(self._is_vtep_reachable, axis=1)
# State is up
df["assertReason"] += df.apply(
lambda x: ['interface is down']
if x['type'] == "L2" and x['state'] != "up" else [],
axis=1)
devices = df["hostname"].unique().tolist()
ifdf = IfObj(context=self.ctxt) \
.get(namespace=kwargs.get("namespace", ""), hostname=devices)
df = df.merge(
ifdf[['namespace', 'hostname', 'ifname', 'master', 'vlan']],
on=['namespace', 'hostname', 'ifname'],
how='left')
# vxlan interfaces, if defined, for every VNI is part of bridge
# We ensure this is true artificially for NXOS, ignored for JunOS.
df["assertReason"] += df.apply(
lambda x: ['vni not in bridge'] if (x['type'] == "L2" and x[
'ifname'] != '-' and x['master'] != "bridge") else [],
axis=1)
# mac_df = MacsObj(context=self.ctxt) \
# .get(namespace=kwargs.get("namespace", ""),
# macaddr=["00:00:00:00:00:00"])
# # Assert that we have HER for every remote VTEP
# df['assertReason'] += df.apply(self._is_her_good,
# args=(mac_df, ), axis=1)
# Fill out the assert column
df['assert'] = df.apply(lambda x: 'pass'
if not len(x.assertReason) else 'fail',
axis=1)
return df[['namespace', 'hostname', 'vni', 'type',
'assertReason', 'assert', 'timestamp']].explode(column='assertReason') \
.fillna({'assertReason': '-'})