def test_interfaces(ansible_adhoc, testbed):
"""compare the interfaces between observed states and target state"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
host_facts = ans_host.setup()['ansible_facts']
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
verify_port(host_facts, mg_facts['minigraph_portchannels'].keys())
for k, v in mg_facts['minigraph_portchannels'].items():
verify_port(host_facts, v['members'])
# verify no ipv4 address for each port channel member
for member in v['members']:
ans_ifname = "ansible_%s" % member
assert not host_facts[ans_ifname].has_key("ipv4")
verify_port(host_facts, mg_facts['minigraph_vlans'].keys())
for k, v in mg_facts['minigraph_vlans'].items():
verify_port(host_facts, v['members'])
# verify no ipv4 address for each vlan member
for member in v['members']:
ans_ifname = "ansible_%s" % member
assert not host_facts[ans_ifname].has_key("ipv4")
verify_ip_address(host_facts, mg_facts['minigraph_portchannel_interfaces'])
verify_ip_address(host_facts, mg_facts['minigraph_vlan_interfaces'])
verify_ip_address(host_facts, mg_facts['minigraph_lo_interfaces'])
def test_snmp_interfaces(ansible_adhoc, duthost, creds):
"""compare the bgp facts between observed states and target state"""
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = duthost.host.options['inventory_manager'].get_host(
duthost.hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(
host=hostip, version="v2c",
community=creds["snmp_rocommunity"])['ansible_facts']
config_facts = duthost.config_facts(host=duthost.hostname,
source="persistent")['ansible_facts']
snmp_ifnames = [
v['name'] for k, v in snmp_facts['snmp_interfaces'].items()
]
print snmp_ifnames
# Verify all physical ports in snmp interface list
for _, alias in config_facts['port_name_to_alias_map'].items():
assert alias in snmp_ifnames
# Verify all port channels in snmp interface list
for po_name in config_facts.get('PORTCHANNEL', {}):
assert po_name in snmp_ifnames
# Verify management port in snmp interface list
for name in config_facts.get('MGMT_INTERFACE', {}):
assert name in snmp_ifnames
def test_show_platform_syseeprom(localhost, ansible_adhoc, testbed):
"""
@summary: Check output of 'show platform syseeprom'
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
logging.info("Check output of '%s'" % CMD_PLATFORM_SYSEEPROM)
platform_info = parse_platform_summary(
ans_host.command(CMD_PLATFORM_SUMMARY)["stdout_lines"])
show_output = ans_host.command(CMD_PLATFORM_SYSEEPROM)
assert show_output[
"rc"] == 0, "Run command '%s' failed" % CMD_PLATFORM_SYSEEPROM
if platform_info["asic"] in ["mellanox"]:
expected_fields = [
"Product Name", "Part Number", "Serial Number", "Base MAC Address",
"Manufacture Date", "Device Version", "MAC Addresses",
"Manufacturer", "Vendor Extension", "ONIE Version", "CRC-32"
]
utility_cmd = "sudo python -c \"import imp; \
m = imp.load_source('eeprom', '/usr/share/sonic/device/%s/plugins/eeprom.py'); \
t = m.board('board', '', '', ''); e = t.read_eeprom(); t.decode_eeprom(e)\"" % platform_info[
"platform"]
utility_cmd_output = ans_host.command(utility_cmd)
for field in expected_fields:
assert show_output["stdout"].find(
field) >= 0, "Expected field %s is not found" % field
assert utility_cmd_output["stdout"].find(
field) >= 0, "Expected field %s is not found" % field
for line in utility_cmd_output["stdout_lines"]:
assert line in show_output["stdout"], \
"Line %s is not found in output of '%s'" % (line, CMD_PLATFORM_SYSEEPROM)
def test_lldp(localhost, ansible_adhoc, testbed):
""" verify the LLDP message on DUT """
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
lldp_facts = ans_host.lldp()['ansible_facts']
minigraph_lldp_nei = {}
for k, v in mg_facts['minigraph_neighbors'].items():
if 'server' not in v['name'].lower():
minigraph_lldp_nei[k] = v
# Verify LLDP information is available on most interfaces
assert len(lldp_facts['lldp']) > len(minigraph_lldp_nei) * 0.8
for k, v in lldp_facts['lldp'].items():
if k == 'eth0':
continue
# Compare the LLDP neighbor name with minigraph neigbhor name (exclude the management port)
assert v['chassis']['name'] == minigraph_lldp_nei[k]['name']
# Compare the LLDP neighbor interface with minigraph neigbhor interface (exclude the management port)
assert v['port']['ifname'] == mg_facts['minigraph_neighbors'][k][
'port']
def run_test(ansible_adhoc,
testbed,
conn_graph_facts,
leaf_fanouts,
is_pfc=True,
pause_time=65535):
"""
@Summary: Run test for Ethernet flow control (FC) or priority-based flow control (PFC)
@param ansible_adhoc: Fixture provided by the pytest-ansible package. Source of the various device objects. It is
mandatory argument for the class constructors.
@param testbed: Testbed information
@param conn_graph_facts: Testbed topology connectivity information
@param leaf_fanouts: Leaf fanout switches
@param is_pfc: If this test is for PFC?
@param pause_time: Pause time quanta (0-65535) in the frame. 0 means unpause.
"""
setup_testbed(ansible_adhoc, testbed, leaf_fanouts)
conn_facts = conn_graph_facts['device_conn']
dut_hostname = testbed['dut']
dut_ans = AnsibleHost(ansible_adhoc, dut_hostname)
int_status = dut_ans.show_interface(
command="status")['ansible_facts']['int_status']
""" We only test active physical interfaces """
active_phy_intfs = [intf for intf in int_status if \
intf.startswith('Ethernet') and \
int_status[intf]['admin_state'] == 'up' and \
int_status[intf]['oper_state'] == 'up']
""" Generate PFC or FC packets for active physical interfaces """
for intf in active_phy_intfs:
peer_device = conn_facts[intf]['peerdevice']
peer_port = conn_facts[intf]['peerport']
peer_port_name = eos_to_linux_intf(peer_port)
peerdev_ans = AnsibleHost(ansible_adhoc, peer_device)
if is_pfc:
for priority in range(PRIO_COUNT):
cmd = "sudo python %s -i %s -p %d -t %d -n %d" % (
PFC_GEN_FILE_DEST, peer_port_name, 2**
priority, pause_time, PKT_COUNT)
peerdev_ans.command(cmd)
else:
cmd = "sudo python %s -i %s -g -t %d -n %d" % (
PFC_GEN_FILE_DEST, peer_port_name, pause_time, PKT_COUNT)
peerdev_ans.command(cmd)
""" SONiC takes some time to update counters in database """
time.sleep(5)
""" Check results """
counter_facts = dut_ans.sonic_pfc_counters(method="get")['ansible_facts']
for intf in active_phy_intfs:
if is_pfc:
assert counter_facts[intf]['Rx'] == [str(PKT_COUNT)] * PRIO_COUNT
else:
assert counter_facts[intf]['Rx'] == ['0'] * PRIO_COUNT
def test_snmp_queues(ansible_adhoc, testbed, creds):
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = ans_host.host.options['inventory_manager'].get_host(hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(host=hostip, version="v2c", community=creds["snmp_rocommunity"])['ansible_facts']
for k, v in snmp_facts['snmp_interfaces'].items():
if "Ethernet" in v['description']:
if not v.has_key('queues'):
pytest.fail("port %s does not have queue counters" % v['name'])
def test_show_platform_psustatus(localhost, ansible_adhoc, testbed):
"""
@summary: Check output of 'show platform psustatus'
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
logging.info("Check PSU status using '%s', hostname: %s" %
(CMD_PLATFORM_PSUSTATUS, hostname))
psu_status = ans_host.command(CMD_PLATFORM_PSUSTATUS)
psu_line_pattern = re.compile(r"PSU\s+\d+\s+(OK|NOT OK)")
for line in psu_status["stdout_lines"][2:]:
assert psu_line_pattern.match(line), "Unexpected PSU status output"
def test_lldp_neighbor(localhost, ansible_adhoc, testbed, eos):
""" verify LLDP information on neighbors """
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
res = ans_host.shell(
"docker exec -i lldp lldpcli show chassis | grep \"SysDescr:\" | sed -e 's/^\\s*SysDescr:\\s*//g'"
)
dut_system_description = res['stdout']
lldp_facts = ans_host.lldp()['ansible_facts']
host_facts = ans_host.setup()['ansible_facts']
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
config_facts = ans_host.config_facts(host=hostname,
source="running")['ansible_facts']
nei_meta = config_facts.get('DEVICE_NEIGHBOR_METADATA', {})
for k, v in lldp_facts['lldp'].items():
if k == 'eth0':
# skip test on management interface
continue
try:
hostip = v['chassis']['mgmt-ip']
except:
logger.info(
"Neighbor device {} does not sent management IP via lldp".
format(v['chassis']['name']))
hostip = nei_meta[v['chassis']['name']]['mgmt_addr']
nei_lldp_facts = lhost.lldp_facts(
host=hostip, version='v2c',
community=eos['snmp_rocommunity'])['ansible_facts']
print nei_lldp_facts
neighbor_interface = v['port']['ifname']
# Verify the published DUT system name field is correct
assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface][
'neighbor_sys_name'] == hostname
# Verify the published DUT chassis id field is not empty
assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface]['neighbor_chassis_id'] == \
"0x%s" % (host_facts['ansible_eth0']['macaddress'].replace(':', ''))
# Verify the published DUT system description field is correct
assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface][
'neighbor_sys_desc'] == dut_system_description
# Verify the published DUT port id field is correct
assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface][
'neighbor_port_id'] == mg_facts['minigraph_ports'][k]['alias']
# Verify the published DUT port description field is correct
assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface]['neighbor_port_desc'] == \
"%s:%s" % (mg_facts['minigraph_neighbors'][k]['name'], mg_facts['minigraph_neighbors'][k]['port'])
def test_warm_reboot(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to perform cold reboot and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
asic_type = ans_host.shell("show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
if asic_type in ["mellanox"]:
issu_capability = ans_host.command("show platform mlnx issu")["stdout"]
if "disabled" in issu_capability:
pytest.skip("ISSU is not supported on this DUT, skip this test case")
reboot_and_check(localhost, ans_host, reboot_type="warm")
def test_fib(ansible_adhoc, testbed, duthost, ptfhost, ipv4, ipv6, mtu):
t1_topologies = ['t1', 't1-lag', 't1-64-lag', 't1-64-lag-clet']
t0_topologies = ['t0', 't0-52', 't0-56', 't0-64', 't0-64-32', 't0-116']
hostname = testbed['dut']
testbed_type = testbed['topo']['name']
if testbed_type in t1_topologies:
props = testbed['topo']['properties']['configuration_properties'][
'spine']
props_tor = testbed['topo']['properties']['configuration_properties'][
'tor']
elif testbed_type in t0_topologies:
props = testbed['topo']['properties']['configuration_properties'][
'common']
props_tor = None
else:
py.test.fail('Unknown testbed topology type {}'.format(testbed_type))
ans_host = AnsibleHost(ansible_adhoc, hostname)
mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts']
extra_vars = {
'testbed_type': testbed_type,
'props': props,
'props_tor': props_tor,
'minigraph_portchannels': mg_facts['minigraph_portchannels'],
'minigraph_vlans': mg_facts['minigraph_vlans'],
'minigraph_port_indices': mg_facts['minigraph_port_indices'],
'minigraph_neighbors': mg_facts['minigraph_neighbors']
}
router_mac = duthost.shell(
'sonic-cfggen -d -v \'DEVICE_METADATA.localhost.mac\''
)["stdout_lines"][0].decode("utf-8")
ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars)
logging.info("extra_vars: %s" %
str(ptfhost.host.options['variable_manager'].extra_vars))
log_file = "/tmp/fib_test.FibTest.ipv4.{}.ipv6.{}.{}.log".format(
ipv4, ipv6,
datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))
logging.info("PTF log file: %s" % log_file)
ptfhost.template(src="fib/fib.j2", dest="/root/fib_info.txt")
ptfhost.copy(src="ptftests", dest="/root")
logging.info("run ptf test")
ptf_runner(ptfhost,
"ptftests",
"fib_test.FibTest",
platform_dir="ptftests",
params={
"testbed_type": testbed_type,
"router_mac": router_mac,
"fib_info": "/root/fib_info.txt",
"ipv4": ipv4,
"ipv6": ipv6,
"testbed_mtu": mtu
},
log_file=log_file,
socket_recv_size=16384)
def test_restart_syncd(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to restart the syncd service and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
restart_service_and_check(localhost, ans_host, "syncd")
def setup_testbed(ansible_adhoc, duthost, leaf_fanouts):
"""
@Summary: Set up the duthost, including clearing counters, and copying the PFC generator to the leaf fanout switches.
@param ansible_adhoc: Fixture provided by the pytest-ansible package. Source of the various device objects. It is
mandatory argument for the class constructors.
@param duthost: dut host information
@param leaf_fanouts: Leaf fanout switches
"""
""" Clear PFC counters """
duthost.sonic_pfc_counters(method="clear")
""" Copy the PFC generator to all the leaf fanout switches """
for peer_device in leaf_fanouts:
peerdev_ans = AnsibleHost(ansible_adhoc, peer_device)
file_src = os.path.join(os.path.dirname(__file__),
PFC_GEN_FILE_RELATIVE_PATH)
peerdev_ans.copy(src=file_src, dest=PFC_GEN_FILE_DEST, force=True)
def gen_testbed_t0(ansible_adhoc, testbed):
"""
@Summary: Generate a T0 testbed configuration
@param ansible_adhoc: Fixture provided by the pytest-ansible package. Source of the various device objects. It is
mandatory argument for the class constructors.
@param testbed: Testbed information
@return: Return four values: DUT interfaces, PTF interfaces, PTF IP addresses, and PTF MAC addresses,
"""
dut_hostname = testbed['dut']
dut_ans = AnsibleHost(ansible_adhoc, dut_hostname)
""" Get all the active physical interfaces enslaved to the Vlan """
""" These interfaces are actually server-faced interfaces at T0 """
vlan_members = get_active_vlan_members(dut_ans, dut_hostname)
""" Get Vlan subnet """
vlan_subnet = get_vlan_subnet(dut_ans, dut_hostname)
""" Prefix length to network mask """
vlan_subnet_mask = ipaddress.ip_network(unicode(vlan_subnet, "utf-8")).netmask
""" Generate IP addresses for servers in the Vlan """
vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, len(vlan_members))
""" Generate MAC addresses 00:00:00:00:00:XX for servers in the Vlan """
vlan_mac_addrs = [5 * '00:' + format(k, '02x') for k in random.sample(range(1, 256), len(vlan_members))]
""" Find correspoinding interfaces on PTF """
phy_intfs = get_phy_intfs(dut_ans)
phy_intfs.sort(key=natural_keys)
vlan_members.sort(key=natural_keys)
vlan_members_index = [phy_intfs.index(intf) for intf in vlan_members]
ptf_intfs = ['eth' + str(i) for i in vlan_members_index]
return vlan_members, ptf_intfs, vlan_ip_addrs, vlan_mac_addrs
def ptfhost(ansible_adhoc, testbed):
"""
Shortcut fixture for getting PTF host
"""
hostname = testbed['ptf']
return AnsibleHost(ansible_adhoc, hostname)
def duthost(ansible_adhoc, testbed):
"""
Shortcut fixture for getting DUT host
"""
hostname = testbed['dut']
return AnsibleHost(ansible_adhoc, hostname)
def test_cold_reboot(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to perform cold reboot and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
reboot_and_check(localhost, ans_host, reboot_type="cold")
def ptfadapter(ansible_adhoc, testbed):
"""return ptf test adapter object.
The fixture is module scope, because usually there is not need to
restart PTF nn agent and reinitialize data plane thread on every
test class or test function/method. Session scope should also be Ok,
however if something goes really wrong in one test module it is safer
to restart PTF before proceeding running other test modules
"""
ptfhost = AnsibleHost(ansible_adhoc, testbed['ptf'])
# get the eth interfaces from PTF and initialize ifaces_map
res = ptfhost.command('cat /proc/net/dev')
ifaces = get_ifaces(res['stdout'])
ifaces_map = {
int(ifname.replace(ETH_PFX, '')): ifname
for ifname in ifaces
}
# generate supervisor configuration for ptf_nn_agent
ptfhost.host.options['variable_manager'].extra_vars = {
'device_num': DEFAULT_DEVICE_NUM,
'ptf_nn_port': DEFAULT_PTF_NN_PORT,
'ifaces_map': ifaces_map,
}
ptfhost.template(src='ptfadapter/templates/ptf_nn_agent.conf.ptf.j2',
dest='/etc/supervisor/conf.d/ptf_nn_agent.conf')
# reread configuration and update supervisor
ptfhost.command('supervisorctl reread')
ptfhost.command('supervisorctl update')
with PtfTestAdapter(testbed['ptf_ip'], DEFAULT_PTF_NN_PORT, 0,
len(ifaces_map)) as adapter:
yield adapter
def test_snmp_pfc_counters(ansible_adhoc, duthost, creds):
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = duthost.host.options['inventory_manager'].get_host(
duthost.hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(
host=hostip, version="v2c",
community=creds["snmp_rocommunity"])['ansible_facts']
# Check PFC counters
# Ignore management ports, assuming the names starting with 'eth', eg. eth0
for k, v in snmp_facts['snmp_interfaces'].items():
if "Ethernet" in v['description']:
if not v.has_key('cpfcIfRequests') or \
not v.has_key('cpfcIfIndications') or \
not v.has_key('requestsPerPriority') or \
not v.has_key('indicationsPerPriority'):
pytest.fail("port %s does not have pfc counters" % v['name'])
def test_show_platform_summary(localhost, ansible_adhoc, testbed):
"""
@summary: Check output of 'show platform summary'
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
logging.info("Check output of '%s'" % CMD_PLATFORM_SUMMARY)
platform_summary = ans_host.command(CMD_PLATFORM_SUMMARY)
expected_fields = set(["Platform", "HwSKU", "ASIC"])
actual_fields = set()
for line in platform_summary["stdout_lines"]:
key_value = line.split(":")
assert len(
key_value) == 2, "output format is not 'field_name: field_value'"
assert len(key_value[1]) > 0, "No value for field %s" % key_value[0]
actual_fields.add(line.split(":")[0])
assert actual_fields == expected_fields, \
"Unexpected output fields, actual=%s, expected=%s" % (str(actual_fields), str(expected_fields))
def test_bgp_facts(ansible_adhoc, testbed,duthost):
"""compare the bgp facts between observed states and target state"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
npus = duthost.num_npus()
bgp_facts = ans_host.bgp_facts(num_npus=npus)['ansible_facts']
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
for k, v in bgp_facts['bgp_neighbors'].items():
# Verify bgp sessions are established
assert v['state'] == 'established'
# Verify locat ASNs in bgp sessions
assert v['local AS'] == mg_facts['minigraph_bgp_asn']
for v in mg_facts['minigraph_bgp']:
# Compare the bgp neighbors name with minigraph bgp neigbhors name
assert v['name'] == bgp_facts['bgp_neighbors'][v['addr'].lower()]['description']
# Compare the bgp neighbors ASN with minigraph
assert v['asn'] == bgp_facts['bgp_neighbors'][v['addr'].lower()]['remote AS']
def test_snmp_interfaces(ansible_adhoc, testbed, creds):
"""compare the bgp facts between observed states and target state"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = ans_host.host.options['inventory_manager'].get_host(hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(host=hostip, version="v2c", community=creds["snmp_rocommunity"])['ansible_facts']
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
snmp_ifnames = [ v['name'] for k, v in snmp_facts['snmp_interfaces'].items() ]
print snmp_ifnames
# Verify all physical ports in snmp interface list
for k, v in mg_facts['minigraph_ports'].items():
assert mg_facts['minigraph_port_name_to_alias_map'][k] in snmp_ifnames
# Verify all port channels in snmp interface list
for k, v in mg_facts['minigraph_portchannels'].items():
assert k in snmp_ifnames
# Verify management port in snmp interface list
assert mg_facts['minigraph_mgmt_interface']['alias'] in snmp_ifnames
print mg_facts['minigraph_mgmt_interface']
def fdb_cleanup(ansible_adhoc, testbed):
""" cleanup FDB before and after test run """
duthost = AnsibleHost(ansible_adhoc, testbed['dut'])
try:
duthost.command('sonic-clear fdb all')
yield
finally:
# in any case clear fdb after test
duthost.command('sonic-clear fdb all')
def test_xcvr_info_in_db(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to verify that xcvrd works as expected by checking transceiver information in DB
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
localhost.command("who")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
logging.info("Check transceiver status")
check_transceiver_status(ans_host, interfaces)
def test_reload_configuration(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to reload the configuration and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
ans_host.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = ans_host.shell(
"show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Reload configuration")
ans_host.command("sudo config reload -y")
logging.info("Wait until all critical services are fully started")
check_critical_services(ans_host)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, ans_host, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
time.sleep(60)
check_interface_status(ans_host, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(ans_host, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(ans_host)
logging.info("Check sysfs")
check_sysfs(ans_host)
def test_snmp_cpu(ansible_adhoc, duthost, creds):
"""
Test SNMP CPU Utilization
- Pulls CPU usage via shell commans
- Polls SNMP for CPU usage
- Difference should be < 2% (allowing float->int rounding on each result)
TODO: abstract the snmp OID by SKU
"""
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = duthost.host.options['inventory_manager'].get_host(
duthost.hostname).vars['ansible_host']
host_facts = duthost.setup()['ansible_facts']
host_vcpus = int(host_facts['ansible_processor_vcpus'])
# Gather facts with SNMP version 2
snmp_facts = lhost.snmp_facts(host=hostip,
version="v2c",
community=creds["snmp_rocommunity"],
is_dell=True)['ansible_facts']
assert int(snmp_facts['ansible_ChStackUnitCpuUtil5sec'])
try:
for i in range(host_vcpus):
duthost.shell("nohup yes > /dev/null 2>&1 & sleep 1")
# Wait for load to reflect in SNMP
time.sleep(20)
# Gather facts with SNMP version 2
snmp_facts = lhost.snmp_facts(host=hostip,
version="v2c",
community=creds["snmp_rocommunity"],
is_dell=True)['ansible_facts']
# Pull CPU utilization via shell
# Explanation: Run top command with 2 iterations, 5sec delay.
# Discard the first iteration, then grap the CPU line from the second,
# subtract 100% - idle, and round down to integer.
output = duthost.shell(
"top -bn2 -d5 | awk '/^top -/ { p=!p } { if (!p) print }' | awk '/Cpu/ { cpu = 100 - $8 };END { print cpu }' | awk '{printf \"%.0f\",$1}'"
)
print int(snmp_facts['ansible_ChStackUnitCpuUtil5sec'])
print int(output['stdout'])
cpu_diff = abs(
int(snmp_facts['ansible_ChStackUnitCpuUtil5sec']) -
int(output['stdout']))
if cpu_diff > 5:
pytest.fail("cpu diff large than 5%%, %d, %d" %
(int(snmp_facts['ansible_ChStackUnitCpuUtil5sec']),
int(output['stdout'])))
duthost.shell("killall yes")
except:
duthost.shell("killall yes")
raise
def test_check_sfp_status_and_configure_sfp(localhost, ansible_adhoc, testbed):
"""
@summary: Check SFP status and configure SFP
This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the
only configuration is to reset SFP. Commands to be tested:
* sfputil show presence
* show interface transceiver presence
* sfputil show eeprom
* show interface transceiver eeprom
* sfputil reset <interface name>
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
localhost.command("who")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_eeprom = "sudo sfputil show eeprom"
cmd_sfp_reset = "sudo sfputil reset"
cmd_xcvr_presence = "show interface transceiver presence"
cmd_xcvr_eeprom = "show interface transceiver eeprom"
logging.info("Check output of '%s'" % cmd_sfp_presence)
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_xcvr_presence)
xcvr_presence = ans_host.command(cmd_xcvr_presence)
parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_sfp_eeprom)
sfp_eeprom = ans_host.command(cmd_sfp_eeprom)
parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'"
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Check output of '%s'" % cmd_xcvr_eeprom)
xcvr_eeprom = ans_host.command(cmd_xcvr_eeprom)
parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Test '%s <interface name>'" % cmd_sfp_reset)
for intf in conn_graph_facts["device_conn"]:
reset_result = ans_host.command("%s %s" % (cmd_sfp_reset, intf))
assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset,
intf)
time.sleep(120) # Wait some time for SFP to fully recover after reset
logging.info("Check sfp presence again after reset")
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
def test_check_sfp_low_power_mode(localhost, ansible_adhoc, testbed):
"""
@summary: Check SFP low power mode
This case is to use the sfputil tool command to check and set SFP low power mode
* sfputil show lpmode
* sfputil lpmode off
* sfputil lpmode on
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
localhost.command("who")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_show_lpmode = "sudo sfputil show lpmode"
cmd_sfp_set_lpmode = "sudo sfputil lpmode"
logging.info("Check output of '%s'" % cmd_sfp_show_lpmode)
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
original_lpmode = copy.deepcopy(parsed_lpmode)
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
for intf in conn_graph_facts["device_conn"]:
new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on"
lpmode_set_result = ans_host.command(
"%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (
cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
for intf in conn_graph_facts["device_conn"]:
new_lpmode = original_lpmode[intf].lower()
lpmode_set_result = ans_host.command(
"%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (
cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Check sfp presence again after setting lpmode")
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
def test_bgp_speaker(localhost, ansible_adhoc, testbed, ipv4, ipv6, mtu):
"""setup bgp speaker on T0 topology and verify routes advertised
by bgp speaker is received by T0 TOR
"""
hostname = testbed['dut']
ptf_hostname = testbed['ptf']
host = AnsibleHost(ansible_adhoc, hostname)
ptfhost = AnsibleHost(ansible_adhoc, ptf_hostname)
mg_facts = host.minigraph_facts(host=hostname)['ansible_facts']
host_facts = host.setup()['ansible_facts']
res = host.shell("sonic-cfggen -m -d -y /etc/sonic/constants.yml -v \"constants.deployment_id_asn_map[DEVICE_METADATA['localhost']['deployment_id']]\"")
bgp_speaker_asn = res['stdout']
vlan_ips = generate_ips(3, \
"%s/%s" % (mg_facts['minigraph_vlan_interfaces'][0]['addr'], mg_facts['minigraph_vlan_interfaces'][0]['prefixlen']),
[IPAddress(mg_facts['minigraph_vlan_interfaces'][0]['addr'])])
# three speaker ips, two from peer range, another is vlan ip [0]
speaker_ips = generate_ips(2, mg_facts['minigraph_bgp_peers_with_range'][0]['ip_range'][0], [])
speaker_ips.append(vlan_ips[0])
for ip in vlan_ips:
host.command("ip route flush %s/32" % ip.ip)
host.command("ip route add %s/32 dev %s" % (ip.ip, mg_facts['minigraph_vlan_interfaces'][0]['attachto']))
root_dir = "/root"
exabgp_dir = "/root/exabgp"
helper_dir = "/root/helpers"
port_num = [5000, 6000, 7000]
cfnames = ["config_1.ini", "config_2.ini", "config_3.ini"]
vlan_ports = []
for i in range(0, 3):
vlan_ports.append(mg_facts['minigraph_port_indices'][mg_facts['minigraph_vlans'][mg_facts['minigraph_vlan_interfaces'][0]['attachto']]['members'][i]])
ptfhost.file(path=exabgp_dir, state="directory")
ptfhost.file(path=helper_dir, state="directory")
ptfhost.copy(src="bgp_speaker/dump.py", dest=helper_dir)
ptfhost.copy(src="bgp_speaker/http_api.py", dest=helper_dir)
ptfhost.copy(src="bgp_speaker/announce_routes.py", dest=helper_dir)
# deploy config file
extra_vars = \
{ 'helper_dir': helper_dir,
'exabgp_dir': exabgp_dir,
'lo_addr' : mg_facts['minigraph_lo_interfaces'][0]['addr'],
'lo_addr_prefixlen' : mg_facts['minigraph_lo_interfaces'][0]['prefixlen'],
'vlan_addr' : mg_facts['minigraph_vlan_interfaces'][0]['addr'],
'peer_range': mg_facts['minigraph_bgp_peers_with_range'][0]['ip_range'][0],
'announce_prefix': '10.10.10.0/26',
'minigraph_portchannels' : mg_facts['minigraph_portchannels'],
'minigraph_vlans' : mg_facts['minigraph_vlans'],
'minigraph_port_indices' : mg_facts['minigraph_port_indices'],
'peer_asn' : mg_facts['minigraph_bgp_asn'],
'peer_asn' : mg_facts['minigraph_bgp_asn'],
'my_asn' : bgp_speaker_asn,
'vlan_ports' : vlan_ports,
'port_num' : port_num,
'speaker_ips': [str(ip) for ip in speaker_ips],
'vlan_ips': [str(ip) for ip in vlan_ips],
'cfnames': cfnames }
for i in range(0, 3):
extra_vars.update({ 'cidx':i })
extra_vars.update({ 'speaker_ip': str(speaker_ips[i].ip) })
ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars)
ptfhost.template(src="bgp_speaker/config.j2", dest="%s/%s" % (exabgp_dir, cfnames[i]))
# deploy routes
ptfhost.template(src="bgp_speaker/routes.j2", dest="%s/%s" % (exabgp_dir, "routes"))
# deploy start script
ptfhost.template(src="bgp_speaker/start.j2", dest="%s/%s" % (exabgp_dir, "start.sh"), mode="u+rwx")
# kill exabgp
res = ptfhost.shell("pkill exabgp || true")
print res
# start exabgp instance
res = ptfhost.shell("bash %s/start.sh" % exabgp_dir)
print res
time.sleep(10)
# announce route
res = ptfhost.shell("nohup python %s/announce_routes.py %s/routes >/dev/null 2>&1 &" % (helper_dir, exabgp_dir))
print res
# make sure routes announced to dynamic bgp neighbors
time.sleep(60)
bgp_facts = host.bgp_facts()['ansible_facts']
# Verify bgp sessions are established
for k, v in bgp_facts['bgp_neighbors'].items():
assert v['state'] == 'established'
# Verify accepted prefixes of the dynamic neighbors are correct
for ip in speaker_ips:
assert bgp_facts['bgp_neighbors'][str(ip.ip)]['accepted prefixes'] == 1
assert bgp_facts['bgp_neighbors'][str(vlan_ips[0].ip)]['accepted prefixes'] == 1
# Generate route-port map information
ptfhost.template(src="bgp_speaker/bgp_speaker_route.j2", dest="/root/bgp_speaker_route.txt")
ptfhost.copy(src="ptftests", dest=root_dir)
ptf_runner(ptfhost, \
"ptftests",
"fib_test.FibTest",
platform_dir="ptftests",
params={"testbed_type": "t0",
"router_mac": host_facts['ansible_Ethernet0']['macaddress'],
"fib_info": "/root/bgp_speaker_route.txt",
"ipv4": ipv4,
"ipv6": ipv6,
"testbed_mtu": mtu },
log_file="/tmp/bgp_speaker_test.FibTest.log",
socket_recv_size=16384)
res = ptfhost.shell("pkill exabgp || true")
for ip in vlan_ips:
host.command("ip route flush %s/32" % ip.ip)
ptfhost.shell("ip addr flush dev eth{}".format(mg_facts['minigraph_port_indices'][mg_facts['minigraph_vlans'][mg_facts['minigraph_vlan_interfaces'][0]['attachto']]['members'][0]]))
def test_snmp_lldp(duthost, ansible_adhoc, creds):
"""
Test checks for ieee802_1ab MIBs:
- lldpLocalSystemData 1.0.8802.1.1.2.1.3
- lldpLocPortTable 1.0.8802.1.1.2.1.3.7
- lldpLocManAddrTable 1.0.8802.1.1.2.1.3.8
- lldpRemTable 1.0.8802.1.1.2.1.4.1
- lldpRemManAddrTable 1.0.8802.1.1.2.1.4.2
For local data check if every OID has value
For remote values check for availability for at least 80% of minigraph neighbors
(similar to lldp test)
"""
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(host=hostip, version="v2c", community=creds["snmp_rocommunity"])['ansible_facts']
mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts']
print snmp_facts['snmp_lldp']
for k in ['lldpLocChassisIdSubtype', 'lldpLocChassisId', 'lldpLocSysName', 'lldpLocSysDesc']:
assert snmp_facts['snmp_lldp'][k]
assert "No Such Object currently exists" not in snmp_facts['snmp_lldp'][k]
# Check if lldpLocPortTable is present for all ports
for k, v in snmp_facts['snmp_interfaces'].items():
if "Ethernet" in v['name'] or "eth" in v['name']:
for oid in ['lldpLocPortIdSubtype', 'lldpLocPortId', 'lldpLocPortDesc']:
assert v.has_key(oid)
assert "No Such Object currently exists" not in v[oid]
# Check if lldpLocManAddrTable is present
for k in ['lldpLocManAddrLen', \
'lldpLocManAddrIfSubtype', \
'lldpLocManAddrIfId', \
'lldpLocManAddrOID']:
assert snmp_facts['snmp_lldp'][k]
assert "No Such Object currently exists" not in snmp_facts['snmp_lldp'][k]
minigraph_lldp_nei = []
for k, v in mg_facts['minigraph_neighbors'].items():
if "server" not in v['name'].lower():
minigraph_lldp_nei.append(k)
print minigraph_lldp_nei
# Check if lldpRemTable is present
active_intf = []
for k, v in snmp_facts['snmp_interfaces'].items():
if v.has_key("lldpRemChassisIdSubtype") and \
v.has_key("lldpRemChassisId") and \
v.has_key("lldpRemPortIdSubtype") and \
v.has_key("lldpRemPortId") and \
v.has_key("lldpRemPortDesc") and \
v.has_key("lldpRemSysName") and \
v.has_key("lldpRemSysDesc") and \
v.has_key("lldpRemSysCapSupported") and \
v.has_key("lldpRemSysCapEnabled"):
active_intf.append(k)
print "lldpRemTable: ", active_intf
assert len(active_intf) >= len(minigraph_lldp_nei) * 0.8
# skip neighbors that do not send chassis information via lldp
lldp_facts = duthost.lldp()['ansible_facts']
nei = [k for k, v in lldp_facts['lldp'].items() if k != 'eth0' and v['chassis'].has_key('mgmt-ip') ]
print "neighbors {} send chassis management IP information".format(nei)
# Check if lldpRemManAddrTable is present
active_intf = []
for k, v in snmp_facts['snmp_interfaces'].items():
if v.has_key("lldpRemManAddrIfSubtype") and \
v.has_key("lldpRemManAddrIfId") and \
v.has_key("lldpRemManAddrOID") and \
v['name'] != 'eth0':
active_intf.append(k)
print "lldpRemManAddrTable: ", active_intf
assert len(active_intf) == len(nei)
def test_snmp_interfaces(ansible_adhoc, testbed, creds):
"""
Test checks for ieee802_1ab MIBs:
- lldpLocalSystemData 1.0.8802.1.1.2.1.3
- lldpLocPortTable 1.0.8802.1.1.2.1.3.7
- lldpLocManAddrTable 1.0.8802.1.1.2.1.3.8
- lldpRemTable 1.0.8802.1.1.2.1.4.1
- lldpRemManAddrTable 1.0.8802.1.1.2.1.4.2
For local data check if every OID has value
For remote values check for availability for at least 80% of minigraph neighbors
(similar to lldp test)
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
lhost = AnsibleHost(ansible_adhoc, 'localhost', True)
hostip = ans_host.host.options['inventory_manager'].get_host(
hostname).vars['ansible_host']
snmp_facts = lhost.snmp_facts(
host=hostip, version="v2c",
community=creds["snmp_rocommunity"])['ansible_facts']
mg_facts = ans_host.minigraph_facts(host=hostname)['ansible_facts']
print snmp_facts['snmp_lldp']
for k in [
'lldpLocChassisIdSubtype', 'lldpLocChassisId', 'lldpLocSysName',
'lldpLocSysDesc'
]:
assert snmp_facts['snmp_lldp'][k]
assert "No Such Object currently exists" not in snmp_facts[
'snmp_lldp'][k]
# Check if lldpLocPortTable is present for all ports
for k, v in snmp_facts['snmp_interfaces'].items():
if "Ethernet" in v['name'] or "eth" in v['name']:
for oid in [
'lldpLocPortNum', 'lldpLocPortIdSubtype', 'lldpLocPortId',
'lldpLocPortDesc'
]:
assert v.has_key(oid)
assert "No Such Object currently exists" not in v[oid]
# Check if lldpLocManAddrTable is present
for k in ['lldpLocManAddrSubtype', \
'lldpLocManAddr', \
'lldpLocManAddrLen', \
'lldpLocManAddrIfSubtype', \
'lldpLocManAddrIfId', \
'lldpLocManAddrOID']:
assert snmp_facts['snmp_lldp'][k]
assert "No Such Object currently exists" not in snmp_facts[
'snmp_lldp'][k]
minigraph_lldp_nei = []
for k, v in mg_facts['minigraph_neighbors'].items():
if "server" not in v['name'].lower():
minigraph_lldp_nei.append(k)
print minigraph_lldp_nei
# Check if lldpRemTable is present
active_intf = []
for k, v in snmp_facts['snmp_interfaces'].items():
if v.has_key("lldpRemTimeMark") and \
v.has_key("lldpRemLocalPortNum") and \
v.has_key("lldpRemIndex") and \
v.has_key("lldpRemChassisIdSubtype") and \
v.has_key("lldpRemChassisId") and \
v.has_key("lldpRemPortIdSubtype") and \
v.has_key("lldpRemPortId") and \
v.has_key("lldpRemPortDesc") and \
v.has_key("lldpRemSysName") and \
v.has_key("lldpRemSysDesc") and \
v.has_key("lldpRemSysCapSupported") and \
v.has_key("lldpRemSysCapEnabled"):
active_intf.append(k)
print active_intf
assert len(active_intf) >= len(minigraph_lldp_nei) * 0.8
# Check if lldpRemManAddrTable is present
active_intf = []
for k, v in snmp_facts['snmp_interfaces'].items():
if v.has_key("lldpRemManAddrSubtype") and \
v.has_key("lldpRemManAddr") and \
v.has_key("lldpRemManAddrIfSubtype") and \
v.has_key("lldpRemManAddrIfId") and \
v.has_key("lldpRemManAddrOID"):
active_intf.append(k)
print active_intf
assert len(active_intf) >= len(minigraph_lldp_nei) * 0.8