def check_rif_on_sup(sup, rif, slot, asic, port):
"""
Checks the router interface entry on the supervisor card.
Args:
sup: duthost for the supervisor card
rif: OID of the router interface to check for.
slot: The slot number the router interface is on.
asic: The asic number the asic is on, or 0 if a single asic card.
port: the name of the port (Ethernet1)
"""
voqdb = VoqDbCli(sup)
slot = str(slot)
if slot.isdigit():
slot_str = "Linecard" + slot
else:
slot_str = slot
asic = str(asic)
if asic.isdigit():
asic_str = "Asic" + asic
else:
asic_str = asic
key = "SYSTEM_INTERFACE|{}|{}|{}".format(slot_str, asic_str, port)
voqdb.get_keys(key)
logger.info("Found key {} on chassisdb on supervisor card".format(key))
def check_voq_neighbor_on_sup(sup, slot, asic, port, neighbor, encap_index,
mac):
"""
Checks the neighbor entry on the supervisor card.
Args:
sup: duthost for the supervisor card
slot: The slot the router interface is on, as in system port table (Slot2).
asic: The asic the router interface is on, as in the system port table (Asic0) .
port: the name of the port (Ethernet1)
neighbor: The IP of the neighbor
encap_index: The encap ID of the neighbor from the local asic db
mac: The MAC address of the neighbor
Raises:
Pytest Failed exception when assertions fail.
"""
voqdb = VoqDbCli(sup)
neigh_key = voqdb.get_neighbor_key_by_ip(neighbor)
logger.info("Neigh key: %s, slotnum: %s", neigh_key, slot)
pytest_assert("|%s|" % slot in neigh_key,
"Slot for %s does not match %s" % (neigh_key, slot))
pytest_assert("|%s:" % port in neigh_key,
"Port for %s does not match %s" % (neigh_key, port))
pytest_assert("|%s|" % asic in neigh_key,
"Asic for %s does not match %s" % (neigh_key, asic))
voqdb.get_and_check_key_value(neigh_key, mac, field="neigh")
voqdb.get_and_check_key_value(neigh_key, encap_index, field="encap_index")
def get_lag_id_from_chassis_db(duthosts):
"""
Get LAG id for a lag form CHASSIS_DB
Args:
duthosts: The duthost fixture.
Returns:
lag_ids <int>: lag id
"""
for sup in duthosts.supervisor_nodes:
voqdb = VoqDbCli(sup)
lag_list = voqdb.get_lag_list()
for lag in lag_list:
if TMP_PC in lag:
lag_id = voqdb.hget_key_value(lag, "lag_id")
logging.info("LAG id for lag {} is {}".format(TMP_PC, lag_id))
return lag_id
pytest.fail("LAG id for lag {} is not preset in CHASSIS_DB".format(TMP_PC))
def verify_lag_member_in_chassis_db(duthosts, members, deleted=False):
"""
verifies lag members for a lag exist in chassis db
cmd = 'redis-cli -h 10.0.5.16 -p 6380 -n 12 KEYS
"*SYSTEM_LAG_MEMBER_TABLE*|PortChannel0051*|Ethernet*"'
"""
for sup in duthosts.supervisor_nodes:
voqdb = VoqDbCli(sup)
lag_member_list = voqdb.get_lag_member_list()
if deleted:
for member in members:
exist = False
pattern = "{}.*{}".format(TMP_PC, member)
for lag_member in lag_member_list:
if re.search(pattern, lag_member):
exist = True
break
if exist:
pytest.fail(
'lag member {} not found in system lag member table {}'
.format(member, lag_member_list))
logging.info(
'lag members {} found in system lag member table {}'.format(
members, lag_member_list))
else:
for member in members:
exist = False
pattern = "{}.*{}".format(TMP_PC, member)
for lag_member in lag_member_list:
if re.search(pattern, lag_member):
exist = True
logging.info(
'lag member {} found in system lag member table {}'
.format(member, lag_member))
break
if not exist:
pytest.fail(
'lag member {} not found in system lag member table {}'
.format(member, lag_member_list))
def get_lag_ids_from_chassis_db(duthosts):
"""
Get lag_ids from CHASSIS_DB
cmd = 'redis-dump -H 10.0.5.16 -p 6380 -d 12 -y -k "*SYSTEM_LAG_TABLE*PortChannel0016"'
Args:
duthosts: The duthost fixture.
Returns:
lag_ids<list>: lag id
"""
lag_ids = list()
for sup in duthosts.supervisor_nodes:
voqdb = VoqDbCli(sup)
lag_list = voqdb.get_lag_list()
for lag in lag_list:
lag_ids.append(voqdb.hget_key_value(lag, "lag_id"))
logging.info("LAG id's preset in CHASSIS_DB are {}".format(lag_ids))
return lag_ids
def check_rif_on_sup(sup, rif, slot, asic, port):
"""
Checks the router interface entry on the supervisor card.
Args:
sup: duthost for the supervisor card
rif: OID of the router interface to check for.
slot: The slot number the router interface is on.
asic: The asic number the asic is on, or 0 if a single asic card.
port: the name of the port (Ethernet1)
"""
voqdb = VoqDbCli(sup)
rif_oid = voqdb.get_router_interface_id(slot, asic, port)
if rif_oid == rif:
logger.info("RIF on sup: %s = %s", rif_oid, rif)
elif rif_oid[-10:-1] == rif[-10:-1]:
logger.warning("RIF on sup is a partial match: %s != %s", rif_oid, rif)
else:
logger.error("RIF on sup does not match: %s != %s" % (rif_oid, rif))
def test_cycle_voq_intf(duthosts, all_cfg_facts, nbrhosts, nbr_macs):
"""
Delete and recreate VOQ interface through config save/load. Verify interface
is removed after configdb reload and then recreated after loading initial minigraph.
Args:
duthosts: The duthosts fixture
all_cfg_facts: all_cfg_facts fixture from voq conftest
nbrhosts: nbrhosts fixture
nbr_macs: nbr_macs fixture from voq conftest
"""
duthost = duthosts.frontend_nodes[0]
for asic in duthost.asics:
cfg_facts = all_cfg_facts[duthost.hostname][
asic.asic_index]['ansible_facts']
check_voq_interfaces(duthosts, duthost, asic, cfg_facts)
intf_asic = duthost.asics[0]
intf_config_facts = duthost.config_facts(
source='persistent', asic_index=intf_asic.asic_index)['ansible_facts']
portchannel = intf_config_facts['PORTCHANNEL'].keys()[0]
portchannel_members = intf_config_facts['PORTCHANNEL'][portchannel].get(
'members')
try:
logger.info(
"remove ethernet from a portchannel to use for interface create")
intf = portchannel_members[0]
logging.info('Deleting lag members {} from lag {} on dut {}'.format(
portchannel_members, portchannel, duthost.hostname))
for member in portchannel_members:
duthost.shell("config portchannel {} member del {} {}".format(
intf_asic.cli_ns_option, portchannel, member))
logger.info("add an IP interface to a former member")
cmd = "config interface {} ip add {} {}".format(
intf_asic.cli_ns_option, intf, str(ADDR))
logger.info("Execute: %s", cmd)
duthost.shell(cmd)
logger.info("Save and reload config")
duthost.shell_cmds(cmds=["config save -y"])
config_reload(duthost, config_source='config_db', wait=600)
logger.info("Check interfaces after add.")
for asic in duthost.asics:
new_cfgfacts = duthost.config_facts(
source='persistent',
asic_index='all')[asic.asic_index]['ansible_facts']
check_voq_interfaces(duthosts, duthost, asic, new_cfgfacts)
logger.info(
"Check interface on supervisor - should be present from chassis db."
)
if duthost.is_multi_asic and len(duthosts.supervisor_nodes) == 0:
sup = duthost
else:
sup = duthosts.supervisor_nodes[0]
voqdb = VoqDbCli(sup)
key = "SYSTEM_INTERFACE|{}|{}|{}".format(
intf_config_facts['DEVICE_METADATA']['localhost']['hostname'],
intf_config_facts['DEVICE_METADATA']['localhost']['asic_name'],
intf)
voqdb.get_keys(key)
logger.info("Remove an IP interface to a former member.")
cmd = "config interface {} ip remove {} {}".format(
intf_asic.cli_ns_option, intf, str(ADDR))
logger.info("Execute: %s", cmd)
duthost.shell(cmd)
logger.info("Save and reload config")
duthost.shell_cmds(cmds=["config save -y"])
config_reload(duthost, config_source='config_db', wait=600)
logger.info("check interface is gone after config reload")
for asic in duthost.asics:
new_cfgfacts = duthost.config_facts(
source='persistent',
asic_index='all')[asic.asic_index]['ansible_facts']
check_voq_interfaces(duthosts, duthost, asic, new_cfgfacts)
with pytest.raises(RedisKeyNotFound):
voqdb.get_keys(key)
logger.info(
"-- Interface {} deleted from chassisdb on supervisor card".format(
key))
finally:
# restore interface from minigraph
logger.info("Restore config from minigraph.")
config_reload(duthost, config_source='minigraph', wait=600)
duthost.shell_cmds(cmds=["config save -y"])
for asic in duthost.asics:
new_cfgfacts = duthost.config_facts(
source='persistent',
asic_index='all')[asic.asic_index]['ansible_facts']
check_voq_interfaces(duthosts, duthost, asic, new_cfgfacts)
def check_voq_interfaces(duthosts, per_host, asic, cfg_facts):
"""
Checks router interfaces on a dut.
Args:
duthosts: The duthosts fixture
per_host: Instance of MultiAsicSonic host to check.
asic: Instance of SonicAsic to check,
cfg_facts: Config facts for the frontend duthost/asic under test
"""
logger.info("Check router interfaces on node: %s, asic: %d",
per_host.hostname, asic.asic_index)
dev_intfs = cfg_facts.get('INTERFACE', {})
voq_intfs = cfg_facts.get('VOQ_INBAND_INTERFACE', [])
dev_sysports = get_device_system_ports(cfg_facts)
rif_ports_in_asicdb = []
# intf_list = get_router_interface_list(dev_intfs)
asicdb = AsicDbCli(asic)
asicdb_rif_table = asicdb.dump(asicdb.ASIC_ROUTERINTF_TABLE)
sys_port_table = asicdb.dump(asicdb.ASIC_SYSPORT_TABLE)
# asicdb_intf_key_list = asicdb.get_router_if_list()
# Check each rif in the asicdb, if it is local port, check VOQ DB for correct RIF.
# If it is on system port, verify slot/asic/port and OID match a RIF in VoQDB
for rif in asicdb_rif_table.keys():
rif_type = asicdb_rif_table[rif]['value'][
"SAI_ROUTER_INTERFACE_ATTR_TYPE"]
if rif_type != "SAI_ROUTER_INTERFACE_TYPE_PORT":
logger.info("Skip this rif: %s, it is not on a port: %s", rif,
rif_type)
continue
else:
portid = asicdb_rif_table[rif]['value'][
"SAI_ROUTER_INTERFACE_ATTR_PORT_ID"]
logger.info("Process RIF %s, Find port with ID: %s", rif, portid)
porttype = asicdb.get_rif_porttype(portid)
logger.info("RIF: %s is of type: %s", rif, porttype)
if porttype == 'hostif':
# find the hostif entry to get the physical port the router interface is on.
hostifkey = asicdb.find_hostif_by_portid(portid)
hostif = asicdb.hget_key_value(hostifkey, 'SAI_HOSTIF_ATTR_NAME')
logger.info("RIF: %s is on local port: %s", rif, hostif)
rif_ports_in_asicdb.append(hostif)
if hostif not in dev_intfs and hostif not in voq_intfs:
pytest.fail(
"Port: %s has a router interface, but it isn't in configdb."
% portid)
# check MTU and ethernet address
pytest_assert(
asicdb_rif_table[rif]['value']["SAI_ROUTER_INTERFACE_ATTR_MTU"]
== cfg_facts['PORT'][hostif]['mtu'],
"MTU for rif %s is not %s" %
(rif, cfg_facts['PORT'][hostif]['mtu']))
intf_mac = get_sonic_mac(per_host, asic.asic_index, hostif)
pytest_assert(
asicdb_rif_table[rif]['value']
["SAI_ROUTER_INTERFACE_ATTR_SRC_MAC_ADDRESS"].lower() ==
intf_mac.lower(), "MAC for rif %s is not %s" % (rif, intf_mac))
sysport_info = {
'slot': cfg_facts['DEVICE_METADATA']['localhost']['hostname'],
'asic': cfg_facts['DEVICE_METADATA']['localhost']['asic_name']
}
if per_host.is_multi_asic and len(duthosts.supervisor_nodes) == 0:
check_rif_on_sup(per_host, sysport_info['slot'],
sysport_info['asic'], hostif)
else:
for sup in duthosts.supervisor_nodes:
check_rif_on_sup(sup, sysport_info['slot'],
sysport_info['asic'], hostif)
elif porttype == 'sysport':
try:
port_output = sys_port_table[
"ASIC_STATE:SAI_OBJECT_TYPE_SYSTEM_PORT:" +
portid]['value']['SAI_SYSTEM_PORT_ATTR_CONFIG_INFO']
except KeyError:
# not a hostif or system port, log error and continue
logger.error("Did not find OID %s in local or system tables" %
portid)
continue
port_data = json.loads(port_output)
for cfg_port in dev_sysports:
if dev_sysports[cfg_port]['system_port_id'] == port_data[
'port_id']:
logger.info("RIF: %s is on remote port: %s", rif, cfg_port)
break
else:
raise AssertionError(
"Did not find OID %s in local or system tables" % portid)
sys_slot, sys_asic, sys_port = cfg_port.split("|")
if per_host.is_multi_asic and len(duthosts.supervisor_nodes) == 0:
check_rif_on_sup(per_host, sys_slot, sys_asic, sys_port)
else:
for sup in duthosts.supervisor_nodes:
check_rif_on_sup(sup, sys_slot, sys_asic, sys_port)
elif porttype == 'port':
# this is the RIF on the inband port.
inband = get_inband_info(cfg_facts)
logger.info("RIF: %s is on local port: %s", rif, inband['port'])
# check MTU and ethernet address
pytest_assert(
asicdb_rif_table[rif]['value']["SAI_ROUTER_INTERFACE_ATTR_MTU"]
== cfg_facts['PORT'][inband['port']]['mtu'],
"MTU for rif %s is not %s" %
(rif, cfg_facts['PORT'][inband['port']]['mtu']))
intf_mac = get_sonic_mac(per_host, asic.asic_index, inband['port'])
pytest_assert(
asicdb_rif_table[rif]['value']
["SAI_ROUTER_INTERFACE_ATTR_SRC_MAC_ADDRESS"].lower() ==
intf_mac.lower(), "MAC for rif %s is not %s" % (rif, intf_mac))
sysport_info = {
'slot': cfg_facts['DEVICE_METADATA']['localhost']['hostname'],
'asic': cfg_facts['DEVICE_METADATA']['localhost']['asic_name']
}
if per_host.is_multi_asic and len(duthosts.supervisor_nodes) == 0:
check_rif_on_sup(per_host, sysport_info['slot'],
sysport_info['asic'], inband['port'])
else:
for sup in duthosts.supervisor_nodes:
check_rif_on_sup(sup, sysport_info['slot'],
sysport_info['asic'], inband['port'])
# TODO: Could be on a LAG
elif porttype == 'lag':
lagid = asicdb.hget_key_value(
"%s:%s" % (AsicDbCli.ASIC_LAG_TABLE, portid),
'SAI_LAG_ATTR_SYSTEM_PORT_AGGREGATE_ID')
logger.info("RIF: %s is on system LAG: %s", rif, lagid)
if per_host.is_multi_asic and len(duthosts.supervisor_nodes) == 0:
voqdb = VoqDbCli(per_host)
else:
voqdb = VoqDbCli(duthosts.supervisor_nodes[0])
systemlagtable = voqdb.dump("SYSTEM_LAG_ID_TABLE")
for lag, sysid in systemlagtable['SYSTEM_LAG_ID_TABLE'][
'value'].iteritems():
if sysid == lagid:
logger.info("System LAG ID %s is portchannel: %s", lagid,
lag)
break
myslot = cfg_facts['DEVICE_METADATA']['localhost']['hostname']
myasic = cfg_facts['DEVICE_METADATA']['localhost']['asic_name']
if lag.startswith("%s|%s" % (myslot, myasic)):
logger.info(
"Lag: %s is a local portchannel with a router interface.",
lag)
(s, a, lagname) = lag.split("|")
pytest_assert(
lagname in cfg_facts['PORTCHANNEL_INTERFACE'],
"RIF Interface %s is in configdb.json but not in asicdb" %
rif)
if per_host.is_multi_asic and len(
duthosts.supervisor_nodes) == 0:
check_rif_on_sup(per_host, myslot, myasic, lagname)
else:
for sup in duthosts.supervisor_nodes:
check_rif_on_sup(sup, myslot, myasic, lagname)
else:
logger.info(
"Lag: %s is a remote portchannel with a router interface.",
lag)
# Verify each RIF in config had a corresponding local port RIF in the asicDB.
for rif in dev_intfs:
if rif not in rif_ports_in_asicdb:
raise AssertionError(
"Interface %s is in configdb.json but not in asicdb" % rif)
logger.info("Interfaces %s are present in configdb.json and asicdb" %
str(dev_intfs.keys()))
def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts):
"""Get parsed FIB information from redis DB for T2 topology.
Args:
duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts
duts_cfg_facts (dict): Running config facts of all DUT hosts.
duts_mg_facts (dict): Minigraph facts of all DUT hosts.
Returns:
dict: Map of prefix to PTF ports that are connected to DUT output ports.
{
'192.168.0.0/21': [],
'192.168.8.0/25': [[58 59] [62 63] [66 67] [70 71]],
'192.168.16.0/25': [[58 59] [62 63] [66 67] [70 71]],
...
'20c0:c2e8:0:80::/64': [[58 59] [62 63] [66 67] [70 71]],
'20c1:998::/64': [[58 59] [62 63] [66 67] [70 71]],
...
}
"""
timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
fib_info = {}
# Collect system neighbors, inband intf and port channel info to resolve ptf ports
# for system neigh or lags.
voq_db = VoqDbCli(duthosts.supervisor_nodes[0])
sys_neigh = {}
for entry in voq_db.dump_neighbor_table():
neigh_key = entry.split('|')
neigh_ip = neigh_key[-1]
sys_neigh[neigh_ip] = {
'duthost_name': neigh_key[-4],
'intf': neigh_key[-2]
}
dut_inband_intfs = {}
dut_port_channels = {}
for duthost in duthosts.frontend_nodes:
cfg_facts = duts_cfg_facts[duthost.hostname]
for asic_cfg_facts in cfg_facts:
dut_inband_intfs.setdefault(duthost.hostname, []).extend(
asic_cfg_facts['VOQ_INBAND_INTERFACE'])
dut_port_channels.setdefault(duthost.hostname, {}).update(
asic_cfg_facts.get('PORTCHANNEL', {}))
for duthost in duthosts.frontend_nodes:
cfg_facts = duts_cfg_facts[duthost.hostname]
mg_facts = duts_mg_facts[duthost.hostname]
for asic_index, asic_cfg_facts in enumerate(cfg_facts):
asic = duthost.asic_instance(asic_index)
asic.shell(
"{} redis-dump -d 0 -k 'ROUTE*' -y > /tmp/fib.{}.txt".format(
asic.ns_arg, timestamp))
duthost.fetch(src="/tmp/fib.{}.txt".format(timestamp),
dest="/tmp/fib")
po = asic_cfg_facts.get('PORTCHANNEL', {})
ports = asic_cfg_facts.get('PORT', {})
with open("/tmp/fib/{}/tmp/fib.{}.txt".format(
duthost.hostname, timestamp)) as fp:
fib = json.load(fp)
for k, v in fib.items():
skip = False
prefix = k.split(':', 1)[1]
ifnames = v['value']['ifname'].split(',')
nh = v['value']['nexthop']
nh_ips = nh.split(',')
oports = []
for idx, ifname in enumerate(ifnames):
if po.has_key(ifname):
# ignore the prefix, if the prefix nexthop is not a frontend port
if 'members' in po[ifname]:
if 'role' in ports[po[ifname]['members'][
0]] and ports[po[ifname]['members']
[0]]['role'] == 'Int':
if len(oports) == 0:
skip = True
else:
oports.append([
str(mg_facts['minigraph_ptf_indices']
[x]) for x in po[ifname]['members']
])
skip = False
else:
if ports.has_key(ifname):
if 'role' in ports[ifname] and ports[ifname][
'role'] == 'Int':
if len(oports) == 0:
skip = True
elif 'role' in ports[ifname] and ports[ifname][
'role'] == 'Inb':
if nh == '0.0.0.0' or nh == '::':
# This is a system or inband neighbor.
neigh_ip = prefix
else:
neigh_ip = nh_ips[idx]
remote_duthost_name = sys_neigh[neigh_ip][
'duthost_name']
remote_neigh_intf = sys_neigh[neigh_ip][
'intf']
# Skip route for inband neighbors.
if remote_neigh_intf in dut_inband_intfs[
remote_duthost_name]:
skip = True
continue
remote_dut_mg_facts = duts_mg_facts[
remote_duthost_name]
if remote_neigh_intf.startswith(
'PortChannel'):
# The nexthop is a system lag.
if dut_port_channels[
remote_duthost_name].has_key(
remote_neigh_intf):
oports.append([
str(remote_dut_mg_facts[
'minigraph_ptf_indices']
[x])
for x in dut_port_channels[
remote_duthost_name]
[remote_neigh_intf]['members']
])
else:
pytest_assert(
False,
"Coundn't find {} in the config of {}"
.format(
remote_neigh_intf,
remote_duthost_name))
else:
# The nexthop is a system neighbor.
oports.append([
str(remote_dut_mg_facts[
'minigraph_ptf_indices']
[remote_neigh_intf])
])
else:
oports.append([
str(mg_facts['minigraph_ptf_indices']
[ifname])
])
skip = False
else:
logger.info(
"Route point to non front panel port {}:{}"
.format(k, v))
skip = True
# skip direct attached subnet
if nh == '0.0.0.0' or nh == '::' or nh == "":
skip = True
if not skip:
if prefix in fib_info:
# Do not add the egress ports if they are already added.
for ops in oports:
if ops not in fib_info[prefix]:
fib_info[prefix].append(ops)
else:
fib_info[prefix] = oports
return fib_info