def check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=True):
tor1 = setup['tor1']
tor1_asn = setup['tor1_asn']
other_vms = setup['other_vms']
# Check route on tor1
logger.info('Check route for prefix {} on {}'.format(route.prefix, tor1))
tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix)
pytest_assert(route.prefix in tor1_route['vrfs']['default']['bgpRouteEntries'].keys(),
'No route for {} found on {}'.format(route.prefix, tor1))
tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
pytest_assert(tor1_route_aspath==route.aspath,
'On {} expected aspath: {}, actual aspath: {}'.format(tor1, route.aspath, tor1_route_aspath))
# Check route on DUT
logger.info('Check route on DUT')
dut_route = duthost.get_route(route.prefix)
if accepted:
pytest_assert(dut_route, 'No route for {} found on DUT'.format(route.prefix))
dut_route_aspath = dut_route['paths'][0]['aspath']['string']
# Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath)
pytest_assert(dut_route_aspath==dut_route_aspath_expected,
'On DUT expected aspath: {}, actual aspath: {}'.format(dut_route_aspath_expected, dut_route_aspath))
else:
pytest_assert(not dut_route, 'Prefix {} should not be accepted by DUT'.format(route.prefix))
# Check route on other VMs
@reset_ansible_local_tmp
def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=None):
logger.info('Check route {} on {}'.format(str(route), node))
dut_asn = setup['dut_asn']
tor1_asn = setup['tor1_asn']
vm_route = nbrhosts[node]['host'].get_route(route.prefix)
vm_route = {'failed': False}
vm_route['tor_route'] = vm_route
if accepted:
if route.prefix not in vm_route['vrfs']['default']['bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route for {} found on {}'.format(route.prefix, node)
else:
tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
# Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
tor_route_aspath_expected = '{} {} {}'.format(dut_asn, tor1_asn, route.aspath)
if tor_route_aspath != tor_route_aspath_expected:
vm_route['failed'] = True
vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\
.format(node, tor_route_aspath_expected, tor_route_aspath)
else:
if route.prefix in vm_route['vrfs']['default']['bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route {} expected on {}'.format(route.prefix, node)
vm_route['message'] = 'Checking route {} on {} passed'.format(str(route), node)
results[node] = vm_route
results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120)
failed_results = {}
for node, result in results.items():
if result['failed']:
failed_results[node] = result
pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\
.format(str(route), json.dumps(failed_results, indent=2)))
def check_routes_on_dut(self, duthost, namespace):
for prefixes in PREFIX_LISTS.values():
for prefix in prefixes:
dut_route = duthost.get_route(prefix, namespace)
pytest_assert(dut_route,
'Route {} is not found on DUT'.format(prefix))
def _simulator_clear_flap_counter(interface_name):
server_url = url(action=CLEAR_FLAP_COUNTER)
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
mbr_index = mg_facts['minigraph_ptf_indices'][interface_name]
data = {"port_to_clear": str(mbr_index)}
pytest_assert(_post(server_url, data), "Failed to clear flap counter for all ports")
def run_pfc_test(api,
testbed_config,
port_config_list,
conn_data,
fanout_data,
duthost,
dut_port,
global_pause,
pause_prio_list,
test_prio_list,
bg_prio_list,
prio_dscp_map,
test_traffic_pause):
"""
Run a PFC test
Args:
api (obj): IXIA session
testbed_config (obj): testbed L1/L2/L3 configuration
port_config_list (list): list of port configuration
conn_data (dict): the dictionary returned by conn_graph_fact.
fanout_data (dict): the dictionary returned by fanout_graph_fact.
duthost (Ansible host instance): device under test
dut_port (str): DUT port to test
global_pause (bool): if pause frame is IEEE 802.3X pause
pause_prio_list (list): priorities to pause for pause frames
test_prio_list (list): priorities of test flows
bg_prio_list (list): priorities of background flows
prio_dscp_map (dict): Priority vs. DSCP map (key = priority).
test_traffic_pause (bool): if test flows are expected to be paused
Returns:
N/A
"""
pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config')
stop_pfcwd(duthost)
disable_packet_aging(duthost)
""" Get the ID of the port to test """
port_id = get_dut_port_id(dut_hostname=duthost.hostname,
dut_port=dut_port,
conn_data=conn_data,
fanout_data=fanout_data)
pytest_assert(port_id is not None,
'Fail to get ID for port {}'.format(dut_port))
""" Rate percent must be an integer """
test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT / len(test_prio_list))
bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT / len(bg_prio_list))
""" Generate traffic config """
flows = __gen_traffic(testbed_config=testbed_config,
port_config_list=port_config_list,
port_id=port_id,
duthost=duthost,
pause_flow_name=PAUSE_FLOW_NAME,
global_pause=global_pause,
pause_prio_list=pause_prio_list,
test_flow_name=TEST_FLOW_NAME,
test_flow_prio_list=test_prio_list,
test_flow_rate_percent=test_flow_rate_percent,
bg_flow_name=BG_FLOW_NAME,
bg_flow_prio_list=bg_prio_list,
bg_flow_rate_percent=bg_flow_rate_percent,
data_flow_dur_sec=DATA_FLOW_DURATION_SEC,
data_flow_delay_sec=DATA_FLOW_DELAY_SEC,
data_pkt_size=DATA_PKT_SIZE,
prio_dscp_map=prio_dscp_map)
""" Tgen config = testbed config + flow config """
config = testbed_config
config.flows = flows
"""
import json
config_json = json.dumps(config, indent=2, default=lambda x: x.__dict__)
print(config_json)
"""
all_flow_names = [flow.name for flow in flows]
data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name]
""" Run traffic """
flow_stats = __run_traffic(api=api,
config=config,
data_flow_names=data_flow_names,
all_flow_names=all_flow_names,
exp_dur_sec=DATA_FLOW_DURATION_SEC+DATA_FLOW_DELAY_SEC)
speed_str = config.layer1[0].speed
speed_gbps = int(speed_str.split('_')[1])
""" Verify experiment results """
__verify_results(rows=flow_stats,
duthost=duthost,
pause_flow_name=PAUSE_FLOW_NAME,
test_flow_name=TEST_FLOW_NAME,
bg_flow_name=BG_FLOW_NAME,
data_flow_dur_sec=DATA_FLOW_DURATION_SEC,
test_flow_rate_percent=test_flow_rate_percent,
bg_flow_rate_percent=bg_flow_rate_percent,
data_pkt_size=DATA_PKT_SIZE,
speed_gbps=speed_gbps,
test_flow_pause=test_traffic_pause,
tolerance=TOLERANCE_THRESHOLD)
def verify_port(host_facts, ports):
for port in ports:
pytest_assert(host_facts[port]['active'],
"interface {} is not active".format(port))
def test_pfc_pause_single_lossy_prio_reboot(
snappi_api, snappi_testbed_config, conn_graph_facts,
fanout_graph_facts, localhost, duthosts, rand_one_dut_hostname,
rand_one_dut_portname_oper_up, rand_lossy_prio, all_prio_list,
prio_dscp_map, reboot_type):
"""
Test if PFC will impact a single lossy priority after various kinds of reboots
Args:
snappi_api (pytest fixture): SNAPPI session
snappi_testbed_config (pytest fixture): testbed configuration information
conn_graph_facts (pytest fixture): connection graph
fanout_graph_facts (pytest fixture): fanout graph
localhost (pytest fixture): localhost handle
duthosts (pytest fixture): list of DUTs
rand_one_dut_hostname (str): hostname of DUT
rand_one_dut_portname_oper_up (str): port to test, e.g., 's6100-1|Ethernet0'
rand_lossy_prio (str): lossy priority to test, e.g., 's6100-1|2'
all_prio_list (pytest fixture): list of all the priorities
prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority).
reboot_type (str): reboot type to be issued on the DUT
Returns:
N/A
"""
dut_hostname, dut_port = rand_one_dut_portname_oper_up.split('|')
dut_hostname2, lossy_prio = rand_lossy_prio.split('|')
pytest_require(rand_one_dut_hostname == dut_hostname == dut_hostname2,
"Priority and port are not mapped to the expected DUT")
testbed_config, port_config_list = snappi_testbed_config
duthost = duthosts[rand_one_dut_hostname]
lossy_prio = int(lossy_prio)
pause_prio_list = [lossy_prio]
test_prio_list = [lossy_prio]
bg_prio_list = [p for p in all_prio_list]
bg_prio_list.remove(lossy_prio)
logger.info("Issuing a {} reboot on the dut {}".format(
reboot_type, duthost.hostname))
reboot(duthost, localhost, reboot_type=reboot_type)
logger.info("Wait until the system is stable")
pytest_assert(
wait_until(300, 20, 0, duthost.critical_services_fully_started),
"Not all critical services are fully started")
run_pfc_test(api=snappi_api,
testbed_config=testbed_config,
port_config_list=port_config_list,
conn_data=conn_graph_facts,
fanout_data=fanout_graph_facts,
duthost=duthost,
dut_port=dut_port,
global_pause=False,
pause_prio_list=pause_prio_list,
test_prio_list=test_prio_list,
bg_prio_list=bg_prio_list,
prio_dscp_map=prio_dscp_map,
test_traffic_pause=False)
def testbed_config(conn_graph_facts, fanout_graph_facts, duthost):
ixia_fanout = get_peer_ixia_chassis(conn_data=conn_graph_facts,
dut_hostname=duthost.hostname)
pytest_require(ixia_fanout is not None,
skip_message="Cannot find the peer IXIA chassis")
ixia_fanout_id = list(fanout_graph_facts.keys()).index(ixia_fanout)
ixia_fanout_list = IxiaFanoutManager(fanout_graph_facts)
ixia_fanout_list.get_fanout_device_details(device_number=ixia_fanout_id)
ixia_ports = ixia_fanout_list.get_ports(peer_device=duthost.hostname)
pytest_require(len(ixia_ports) >= 2,
skip_message="The test requires at least two ports")
rx_id = 0
tx_id = 1
rx_port_location = get_tgen_location(ixia_ports[rx_id])
tx_port_location = get_tgen_location(ixia_ports[tx_id])
rx_port_speed = int(ixia_ports[rx_id]['speed'])
tx_port_speed = int(ixia_ports[tx_id]['speed'])
pytest_require(rx_port_speed == tx_port_speed,
skip_message="Two ports should have the same speed")
""" L1 configuration """
rx_port = Port(name='Rx Port', location=rx_port_location)
tx_port = Port(name='Tx Port', location=tx_port_location)
pfc = Ieee8021qbb(pfc_delay=1,
pfc_class_0=0,
pfc_class_1=1,
pfc_class_2=2,
pfc_class_3=3,
pfc_class_4=4,
pfc_class_5=5,
pfc_class_6=6,
pfc_class_7=7)
flow_ctl = FlowControl(choice=pfc)
auto_negotiation = AutoNegotiation(link_training=True, rs_fec=True)
l1_config = Layer1(name='L1 config',
speed='speed_%d_gbps' % (rx_port_speed / 1000),
auto_negotiate=False,
auto_negotiation=auto_negotiation,
flow_control=flow_ctl,
port_names=[tx_port.name, rx_port.name])
config = Config(ports=[tx_port, rx_port],
layer1=[l1_config],
options=Options(PortOptions(location_preemption=True)))
vlan_subnet = get_vlan_subnet(duthost)
pytest_assert(vlan_subnet is not None,
"Fail to get Vlan subnet information")
vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, 2)
gw_addr = vlan_subnet.split('/')[0]
prefix = vlan_subnet.split('/')[1]
tx_port_ip = vlan_ip_addrs[0]
rx_port_ip = vlan_ip_addrs[1]
tx_gateway_ip = gw_addr
rx_gateway_ip = gw_addr
""" L2/L3 configuration """
tx_ipv4 = Ipv4(name='Tx Ipv4',
address=Pattern(tx_port_ip),
prefix=Pattern(prefix),
gateway=Pattern(tx_gateway_ip),
ethernet=Ethernet(name='Tx Ethernet'))
config.devices.append(
Device(name='Tx Device',
device_count=1,
container_name=tx_port.name,
choice=tx_ipv4))
rx_ipv4 = Ipv4(name='Rx Ipv4',
address=Pattern(rx_port_ip),
prefix=Pattern(prefix),
gateway=Pattern(rx_gateway_ip),
ethernet=Ethernet(name='Rx Ethernet'))
config.devices.append(
Device(name='Rx Device',
device_count=1,
container_name=rx_port.name,
choice=rx_ipv4))
return config
def dutQosConfig(self, duthosts, rand_one_dut_hostname, dutConfig,
ingressLosslessProfile, ingressLossyProfile,
egressLosslessProfile, egressLossyProfile,
sharedHeadroomPoolSize, tbinfo):
"""
Prepares DUT host QoS configuration
Args:
duthost (AnsibleHost): Device Under Test (DUT)
ingressLosslessProfile (Fxiture): ingressLosslessProfile fixture is required to run prior to collecting
QoS configuration
Returns:
QoSConfig (dict): Map containing DUT host QoS configuration
"""
duthost = duthosts[rand_one_dut_hostname]
mgFacts = duthost.get_extended_minigraph_facts(tbinfo)
pytest_assert("minigraph_hwsku" in mgFacts, "Could not find DUT SKU")
profileName = ingressLosslessProfile["profileName"]
if self.isBufferInApplDb(duthost):
profile_pattern = "^BUFFER_PROFILE_TABLE\:pg_lossless_(.*)_profile$"
else:
profile_pattern = "^BUFFER_PROFILE\|pg_lossless_(.*)_profile"
m = re.search(profile_pattern, profileName)
pytest_assert(m.group(1), "Cannot find port speed/cable length")
portSpeedCableLength = m.group(1)
qosConfigs = {}
with open(r"qos/files/qos.yml") as file:
qosConfigs = yaml.load(file, Loader=yaml.FullLoader)
vendor = duthost.facts["asic_type"]
hostvars = duthost.host.options['variable_manager']._hostvars[
duthost.hostname]
dutAsic = None
for asic in self.SUPPORTED_ASIC_LIST:
vendorAsic = "{0}_{1}_hwskus".format(vendor, asic)
if vendorAsic in hostvars.keys(
) and mgFacts["minigraph_hwsku"] in hostvars[vendorAsic]:
dutAsic = asic
break
pytest_assert(dutAsic, "Cannot identify DUT ASIC type")
if isMellanoxDevice(duthost):
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "files/mellanox/")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
import qos_param_generator
qpm = qos_param_generator.QosParamMellanox(
qosConfigs['qos_params']['mellanox'], dutAsic,
portSpeedCableLength, dutConfig, ingressLosslessProfile,
ingressLossyProfile, egressLosslessProfile, egressLossyProfile,
sharedHeadroomPoolSize)
qosParams = qpm.run()
else:
qosParams = qosConfigs['qos_params'][dutAsic]
yield {
"param": qosParams,
"portSpeedCableLength": portSpeedCableLength,
}
def send_and_check_mirror_packets(self,
setup,
mirror_session,
ptfadapter,
duthost,
mirror_packet,
src_port=None,
dest_ports=None,
expect_recv=True,
valid_across_namespace=True):
if not src_port:
src_port = self._get_random_src_port(setup)
if not dest_ports:
dest_ports = [self._get_monitor_port(setup, mirror_session, duthost)]
# In Below logic idea is to send traffic in such a way so that mirror traffic
# will need to go across namespaces and within namespace. If source and mirror destination
# namespace are different then traffic mirror will go across namespace via (backend asic)
# else via same namespace(asic)
src_port_namespace = self._get_port_namespace(setup, int(src_port))
dest_ports_namespace = self._get_port_namespace(setup,int (dest_ports[0]))
src_port_set = set()
# Some of test scenario are not valid across namespaces so test will explicltly pass
# valid_across_namespace as False (default is True)
if valid_across_namespace == True or src_port_namespace == dest_ports_namespace:
src_port_set.add(src_port)
# To verify same namespace mirroring we will add destination port also to the Source Port Set
if src_port_namespace != dest_ports_namespace:
src_port_set.add(dest_ports[0])
expected_mirror_packet_with_ttl = BaseEverflowTest.get_expected_mirror_packet(mirror_session,
setup,
duthost,
mirror_packet,
True)
expected_mirror_packet_without_ttl = BaseEverflowTest.get_expected_mirror_packet(mirror_session,
setup,
duthost,
mirror_packet,
False)
# Loop through Source Port Set and send traffic on each source port of the set
for src_port in src_port_set:
expected_mirror_packet = expected_mirror_packet_with_ttl \
if self._get_port_namespace(setup, int(src_port)) == dest_ports_namespace else expected_mirror_packet_without_ttl
ptfadapter.dataplane.flush()
testutils.send(ptfadapter, src_port, mirror_packet)
if expect_recv:
time.sleep(STABILITY_BUFFER)
_, received_packet = testutils.verify_packet_any_port(
ptfadapter,
expected_mirror_packet,
ports=dest_ports
)
logging.info("Received packet: %s", packet.Ether(received_packet).summary())
inner_packet = self._extract_mirror_payload(received_packet, len(mirror_packet))
logging.info("Received inner packet: %s", inner_packet.summary())
inner_packet = Mask(inner_packet)
# For egress mirroring, we expect the DUT to have modified the packet
# before forwarding it. Specifically:
#
# - In L2 the SMAC and DMAC will change.
# - In L3 the TTL and checksum will change.
#
# We know what the TTL and SMAC should be after going through the pipeline,
# but DMAC and checksum are trickier. For now, update the TTL and SMAC, and
# mask off the DMAC and IP Checksum to verify the packet contents.
if self.mirror_type() == "egress":
mirror_packet[packet.IP].ttl -= 1
mirror_packet[packet.Ether].src = setup["router_mac"]
inner_packet.set_do_not_care_scapy(packet.Ether, "dst")
inner_packet.set_do_not_care_scapy(packet.IP, "chksum")
logging.info("Expected inner packet: %s", mirror_packet.summary())
pytest_assert(inner_packet.pkt_match(mirror_packet), "Mirror payload does not match received packet")
else:
testutils.verify_no_packet_any(ptfadapter, expected_mirror_packet, dest_ports)
def test_power_off_reboot(duthosts, enum_rand_one_per_hwsku_hostname,
localhost, conn_graph_facts, xcvr_skip_list,
pdu_controller, power_off_delay):
"""
@summary: This test case is to perform reboot via powercycle and check platform status
@param duthost: Fixture for DUT AnsibleHost object
@param localhost: Fixture for interacting with localhost through ansible
@param conn_graph_facts: Fixture parse and return lab connection graph
@param xcvr_skip_list: list of DUT's interfaces for which transeiver checks are skipped
@param pdu_controller: The python object of psu controller
@param power_off_delay: Pytest parameter. The delay between turning off and on the PSU
"""
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
UNSUPPORTED_ASIC_TYPE = ["cisco-8000"]
if duthost.facts["asic_type"] in UNSUPPORTED_ASIC_TYPE:
pytest.skip(
"Skipping test_power_off_reboot. Test unsupported on {} platform".
format(duthost.facts["asic_type"]))
pdu_ctrl = pdu_controller
if pdu_ctrl is None:
pytest.skip(
"No PSU controller for %s, skip rest of the testing in this case" %
duthost.hostname)
all_outlets = pdu_ctrl.get_outlet_status()
# If PDU supports returning output_watts, making sure that all outlets has power.
no_power = [
item for item in all_outlets if int(item.get('output_watts', '1')) == 0
]
pytest_assert(not no_power,
"Not all outlets have power output: {}".format(no_power))
# Purpose of this list is to control sequence of turning on PSUs in power off testing.
# If there are 2 PSUs, then 3 scenarios would be covered:
# 1. Turn off all PSUs, turn on PSU1, then check.
# 2. Turn off all PSUs, turn on PSU2, then check.
# 3. Turn off all PSUs, turn on one of the PSU, then turn on the other PSU, then check.
power_on_seq_list = []
if all_outlets:
power_on_seq_list = [[item] for item in all_outlets]
power_on_seq_list.append(all_outlets)
logging.info("Got all power on sequences {}".format(power_on_seq_list))
poweroff_reboot_kwargs = {"dut": duthost}
try:
for power_on_seq in power_on_seq_list:
poweroff_reboot_kwargs["pdu_ctrl"] = pdu_ctrl
poweroff_reboot_kwargs["all_outlets"] = all_outlets
poweroff_reboot_kwargs["power_on_seq"] = power_on_seq
poweroff_reboot_kwargs["delay_time"] = power_off_delay
reboot_and_check(localhost, duthost,
conn_graph_facts["device_conn"][duthost.hostname],
xcvr_skip_list, REBOOT_TYPE_POWEROFF,
_power_off_reboot_helper, poweroff_reboot_kwargs)
except Exception as e:
logging.debug("Restore power after test failure")
for outlet in all_outlets:
logging.debug("turning on {}".format(outlet))
pdu_ctrl.turn_on_outlet(outlet)
# Sleep 120 for dut to boot up
time.sleep(120)
wait_critical_processes(duthost)
raise e
def test_bgp_multipath_relax(tbinfo, duthosts, rand_one_dut_hostname):
duthost = duthosts[rand_one_dut_hostname]
logger.info("Starting test_bgp_multipath_relax on topology {}".format(
tbinfo['topo']['name']))
topo_config = tbinfo['topo']['properties']['configuration']
bgp_v4nei = get_bgp_v4_neighbors_from_minigraph(duthost, tbinfo)
logger.info("bgp_v4nei {}".format(bgp_v4nei))
# get all t0 routers name which has vips defined
dut_t0_neigh = get_t0_neigh(tbinfo, topo_config)
# get t2 neighbors
dut_t2_neigh = get_t2_neigh(tbinfo)
if not dut_t0_neigh:
pytest.fail("Didn't find multipath t0's")
vips_prefix = get_vips_prefix(dut_t0_neigh, topo_config)
logger.info("vips_prefix = {}".format(vips_prefix))
# find all paths of the prefix for test
vips_t0, vips_asn = get_vips_prefix_paths(dut_t0_neigh, vips_prefix,
topo_config)
logger.info("vips_t0: {}, vips_asn: {}".format(vips_t0, vips_asn))
pytest_assert(
(vips_t0 > 1),
"Did not find preconfigured multipath for the vips prefix under test")
# Get the route from the DUT for the prefix
bgp_route = duthost.bgp_route(
prefix=vips_prefix)['ansible_facts']['bgp_route']
logger.info("Bgp route from DUT for prefix {} is {}".format(
vips_prefix, bgp_route))
# Verify found vips prefix entry in Sonic bgp routes
pytest_assert(bgp_route[vips_prefix]['found'] == True,
"BGP route for {} not found".format(vips_prefix))
# Verify total multipath match number of t0 with vips that has prefix for test
pytest_assert(
int(bgp_route[vips_prefix]['path_num']) == len(vips_t0),
"Path number doesnt match the T0s with VIPS")
# verify vips asn in each path of installed BGP vips prefix
for asn in vips_asn:
for aspath in bgp_route[vips_prefix]['aspath']:
pytest_assert((str(asn) in aspath))
# gather one t2 neighbor advertised routes to validate routes advertised to t2 are correct with relaxed multipath
bgp_route_neiadv = duthost.bgp_route(
neighbor=bgp_v4nei[dut_t2_neigh[0]],
direction="adv")['ansible_facts']['bgp_route_neiadv']
logger.info(
"Bgp neighbor adv from DUT for neigh {} and prefix {} is {}".format(
bgp_v4nei[dut_t2_neigh[0]], vips_prefix,
bgp_route_neiadv[vips_prefix]))
# Verify vips prefix in advertised routes to t2 neighbor
pytest_assert(bgp_route_neiadv.has_key(vips_prefix),
"{} is not present in bgp neighbor adv".format(vips_prefix))
# vips prefix path has only 2 hops
pytest_assert((len(bgp_route_neiadv[vips_prefix]['aspath']) == 2),
"vips prefix path doesn't have 2 hops")
pytest_assert((int(bgp_route_neiadv[vips_prefix]['aspath'][0]) in vips_t0 and \
int(bgp_route_neiadv[vips_prefix]['aspath'][1]) in vips_asn), "vips_prefix asn doesnt match with bgp route adv")
def test_ro_user_allowed_command(localhost, duthosts,
enum_rand_one_per_hwsku_hostname,
tacacs_creds, check_tacacs):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
dutip = duthost.mgmt_ip
# Run as RO and use the commands allowed by the sudoers file
commands = {
"cat": [
"sudo cat /var/log/syslog", "sudo cat /var/log/syslog.1",
"sudo cat /var/log/syslog.2.gz"
],
"brctl": ["sudo brctl show"],
"docker": [
"sudo docker exec snmp cat /etc/snmp/snmpd.conf",
'sudo docker images --format "table {% raw %}{{.Repository}}\\t{{.Tag}}\\t{{.ID}}\\t{{.Size}}{% endraw %}"',
"sudo docker ps",
"sudo docker ps -a",
],
"lldpctl": ["sudo lldpctl"],
"vtysh": [
'sudo vtysh -c "show version"',
'sudo vtysh -c "show bgp ipv4 summary json"',
'sudo vtysh -c "show bgp ipv6 summary json"'
],
"rvtysh": [
'sudo rvtysh -c "show ip bgp su"',
'sudo rvtysh -n 0 -c "show ip bgp su"'
],
"decode-syseeprom": ["sudo decode-syseeprom"],
"generate_dump": ['sudo generate_dump -s "5 secs ago"'],
"lldpshow": ["sudo lldpshow"],
"pcieutil": ["sudo pcieutil check"],
"ip": ["sudo ip netns identify 1"],
"ipintutil": [
"sudo ipintutil",
"sudo ipintutil -a ipv6",
"sudo ipintutil -n asic0 -d all",
"sudo ipintutil -n asic0 -d all -a ipv6",
],
"show": [
"show version",
"show interface status",
"show interface portchannel",
"show ip bgp summary",
"show ip bgp neighbors",
"show ip bgp network",
"show ip interface",
"show ipv6 interface",
"show ipv6 bgp neighbors",
"show ipv6 bgp network",
"show lldp table",
],
}
# NOTE: `sudo tail -F /var/log/syslog` will not exit, not posssible to test here
# NOTE: `sudo docker exec bgp cat /etc/quagga/bgpd.conf` can only run on image with quagga
# TODO: some commands need further preparation, will enable when runable directly:
# sudo sensors
# sudo psuutil *
# sudo sfputil show
for command in commands:
if does_command_exist(localhost, dutip, tacacs_creds['tacacs_ro_user'],
tacacs_creds['tacacs_ro_user_passwd'], command):
for subcommand in commands[command]:
allowed = ssh_remote_allow_run(
localhost, dutip, tacacs_creds['tacacs_ro_user'],
tacacs_creds['tacacs_ro_user_passwd'], subcommand)
pytest_assert(allowed,
"command '{}' not authorized".format(subcommand))
else:
logger.info('"{}" not found on DUT, skipping...'.format(command))
dash_allowed = ssh_remote_allow_run(localhost, dutip,
tacacs_creds['tacacs_ro_user'],
tacacs_creds['tacacs_ro_user_passwd'],
'sudo sonic-installer list')
if not dash_allowed:
dash_banned = ssh_remote_ban_run(localhost, dutip,
tacacs_creds['tacacs_ro_user'],
tacacs_creds['tacacs_ro_user_passwd'],
'sudo sonic-installer list')
pytest_assert(
dash_banned,
"command 'sudo sonic-installer list' should be either allowed or banned"
)
underscore_allowed = ssh_remote_allow_run(
localhost, dutip, tacacs_creds['tacacs_ro_user'],
tacacs_creds['tacacs_ro_user_passwd'], 'sudo sonic_installer list')
pytest_assert(
underscore_allowed,
"command 'sudo sonic_installer list' should be allowed if"
" 'sudo sonic-installer list' is banned")
def base_configs(testbed, conn_graph_facts, duthost, lossless_prio_dscp_map,
one_hundred_gbe, start_delay, traffic_duration,
pause_line_rate, traffic_line_rate, pause_frame_type,
frame_size, serializer):
for config in one_hundred_gbe:
start_delay = start_delay * 1000000000.0
bg_dscp_list = [str(prio) for prio in lossless_prio_dscp_map]
test_dscp_list = [str(x) for x in range(64) if x not in bg_dscp_list]
tx = config.ports[0]
rx = config.ports[1]
vlan_subnet = get_vlan_subnet(duthost)
pytest_assert(vlan_subnet is not None,
"Fail to get Vlan subnet information")
vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, 2)
gw_addr = vlan_subnet.split('/')[0]
interface_ip_addr = vlan_ip_addrs[0]
tx_port_ip = vlan_ip_addrs[1]
rx_port_ip = vlan_ip_addrs[0]
tx_gateway_ip = gw_addr
rx_gateway_ip = gw_addr
test_flow_name = 'Test Data'
background_flow_name = 'Background Data'
test_line_rate = traffic_line_rate
background_line_rate = traffic_line_rate
pause_line_rate = pause_line_rate
pytest_assert(
test_line_rate + background_line_rate <= 100,
"test_line_rate + background_line_rate should be less than 100")
######################################################################
# Create TX stack configuration
######################################################################
tx_ipv4 = Ipv4(name='Tx Ipv4',
address=Pattern(tx_port_ip),
prefix=Pattern('24'),
gateway=Pattern(tx_gateway_ip),
ethernet=Ethernet(name='Tx Ethernet'))
tx.devices.append(
Device(name='Tx Device', device_count=1, choice=tx_ipv4))
######################################################################
# Create RX stack configuration
######################################################################
rx_ipv4 = Ipv4(name='Rx Ipv4',
address=Pattern(rx_port_ip),
prefix=Pattern('24'),
gateway=Pattern(rx_gateway_ip),
ethernet=Ethernet(name='Rx Ethernet'))
rx.devices.append(
Device(name='Rx Device', device_count=1, choice=rx_ipv4))
######################################################################
# Traffic configuration Test data
######################################################################
data_endpoint = DeviceTxRx(
tx_device_names=[tx.devices[0].name],
rx_device_names=[rx.devices[0].name],
)
test_dscp = Priority(Dscp(phb=FieldPattern(choice=test_dscp_list)))
test_flow = Flow(name=test_flow_name,
tx_rx=TxRx(data_endpoint),
packet=[
Header(choice=EthernetHeader()),
Header(choice=Ipv4Header(priority=test_dscp))
],
size=Size(frame_size),
rate=Rate('line', test_line_rate),
duration=Duration(
FixedSeconds(seconds=traffic_duration,
delay=start_delay,
delay_unit='nanoseconds')))
config.flows.append(test_flow)
#######################################################################
# Traffic configuration Background data
#######################################################################
background_dscp = Priority(Dscp(phb=FieldPattern(choice=bg_dscp_list)))
background_flow = Flow(
name=background_flow_name,
tx_rx=TxRx(data_endpoint),
packet=[
Header(choice=EthernetHeader()),
Header(choice=Ipv4Header(priority=background_dscp))
],
size=Size(frame_size),
rate=Rate('line', background_line_rate),
duration=Duration(
FixedSeconds(seconds=traffic_duration,
delay=start_delay,
delay_unit='nanoseconds')))
config.flows.append(background_flow)
#######################################################################
# Traffic configuration Pause
#######################################################################
pause_endpoint = PortTxRx(tx_port_name='Rx', rx_port_names=['Rx'])
if (pause_frame_type == 'priority'):
pause = Header(
PfcPause(
dst=FieldPattern(choice='01:80:C2:00:00:01'),
src=FieldPattern(choice='00:00:fa:ce:fa:ce'),
class_enable_vector=FieldPattern(choice='E7'),
pause_class_0=FieldPattern(choice='ffff'),
pause_class_1=FieldPattern(choice='ffff'),
pause_class_2=FieldPattern(choice='ffff'),
pause_class_3=FieldPattern(choice='0'),
pause_class_4=FieldPattern(choice='0'),
pause_class_5=FieldPattern(choice='ffff'),
pause_class_6=FieldPattern(choice='ffff'),
pause_class_7=FieldPattern(choice='ffff'),
))
pause_flow = Flow(name='Pause Storm',
tx_rx=TxRx(pause_endpoint),
packet=[pause],
size=Size(64),
rate=Rate('line', value=100),
duration=Duration(
FixedPackets(packets=0,
delay=0,
delay_unit='nanoseconds')))
elif (pause_frame_type == 'global'):
pause = Header(
EthernetPause(dst=FieldPattern(choice='01:80:C2:00:00:01'),
src=FieldPattern(choice='00:00:fa:ce:fa:ce')))
pause_flow = Flow(name='Pause Storm',
tx_rx=TxRx(pause_endpoint),
packet=[pause],
size=Size(64),
rate=Rate('line', value=pause_line_rate),
duration=Duration(
FixedPackets(packets=0,
delay=0,
delay_unit='nanoseconds')))
else:
pass
config.flows.append(pause_flow)
return one_hundred_gbe
def test_config_db_parameters(duthost):
"""Verifies required telemetry parameters from config_db.
"""
docker_present = verify_telemetry_dockerimage(duthost)
if not docker_present:
pytest.skip("docker-sonic-telemetry is not part of the image")
gnmi = duthost.shell('sonic-db-cli CONFIG_DB HGETALL "TELEMETRY|gnmi"',
module_ignore_errors=False)['stdout_lines']
pytest_assert(gnmi is not None,
"TELEMETRY|gnmi does not exist in config_db")
certs = duthost.shell('sonic-db-cli CONFIG_DB HGETALL "TELEMETRY|certs"',
module_ignore_errors=False)['stdout_lines']
pytest_assert(certs is not None,
"TELEMETRY|certs does not exist in config_db")
d = get_dict_stdout(gnmi, certs)
for key, value in d.items():
if str(key) == "port":
port_expected = str(TELEMETRY_PORT)
pytest_assert(
str(value) == port_expected,
"'port' value is not '{}'".format(port_expected))
if str(key) == "ca_crt":
ca_crt_value_expected = "/etc/sonic/telemetry/dsmsroot.cer"
pytest_assert(
str(value) == ca_crt_value_expected,
"'ca_crt' value is not '{}'".format(ca_crt_value_expected))
if str(key) == "server_key":
server_key_expected = "/etc/sonic/telemetry/streamingtelemetryserver.key"
pytest_assert(
str(value) == server_key_expected,
"'server_key' value is not '{}'".format(server_key_expected))
if str(key) == "server_crt":
server_crt_expected = "/etc/sonic/telemetry/streamingtelemetryserver.cer"
pytest_assert(
str(value) == server_crt_expected,
"'server_crt' value is not '{}'".format(server_crt_expected))
def test_front_panel_admindown_port(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
all_cfg_facts, setup, teardown, nbrhosts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when the DUT port is admin down/up.
Test Steps
* Admin down interface on DUT.
* On local linecard:
* Verify ARP/NDP entries are removed from CLI for neighbors on down port.
* Verify table entries in ASIC, AppDb are removed for addresses on down port.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other
line cards should still be present.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses.
* Verify kernel routes for cleared address are deleted.
* Admin interface up, verify recreation after restarting traffic.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish ARP to all neighbors.
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
intfs, intfs_to_test = pick_ports(cfg_facts)
logger.info("Will test interfaces: %s", intfs_to_test)
for intf in intfs_to_test:
local_ips = [i.split("/")[0] for i in intfs[intf].keys()] # [u'2064:100::2/64', u'100.0.0.2/24']
neighbors = [n for n in neighs if neighs[n]['local_addr'] in local_ips]
logger.info("Testing neighbors: %s on intf: %s", neighbors, intf)
self.localport_admindown(per_host, asic, intf)
try:
check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, neighbors)
finally:
self.localport_adminup(per_host, asic, intf)
for neighbor in neighbors:
sonic_ping(asic, neighbor)
for neighbor in neighbors:
pytest_assert(wait_until(60, 2, check_arptable_state, per_host, asic, neighbor, "REACHABLE"),
"STATE for neighbor {} did not change to reachable".format(neighbor))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, neighbors, nbrhosts, all_cfg_facts, nbr_macs)
def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost,
get_function_conpleteness_level):
# Perform FDB clean up before each test
fdb_cleanup(duthosts, rand_one_dut_hostname)
normalized_level = get_function_conpleteness_level
if normalized_level is None:
normalized_level = "basic"
loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level]
duthost = duthosts[rand_one_dut_hostname]
conf_facts = duthost.config_facts(host=duthost.hostname,
source="persistent")['ansible_facts']
# reinitialize data plane due to above changes on PTF interfaces
ptfadapter.reinit()
router_mac = duthost.facts['router_mac']
port_index_to_name = {
v: k
for k, v in conf_facts['port_index_map'].items()
}
# Only take interfaces that are in ptf topology
ptf_ports_available_in_topo = ptfhost.host.options[
'variable_manager'].extra_vars.get("ifaces_map")
available_ports_idx = []
for idx, name in ptf_ports_available_in_topo.items():
if idx in port_index_to_name and conf_facts['PORT'][
port_index_to_name[idx]].get('admin_status', 'down') == 'up':
available_ports_idx.append(idx)
vlan_table = {}
interface_table = defaultdict(set)
config_portchannels = conf_facts.get('PORTCHANNEL', {})
# if DUT has more than one VLANs, use the first vlan
name = conf_facts['VLAN'].keys()[0]
vlan = conf_facts['VLAN'][name]
vlan_id = int(vlan['vlanid'])
vlan_table[vlan_id] = []
for ifname in conf_facts['VLAN_MEMBER'][name].keys():
if 'tagging_mode' not in conf_facts['VLAN_MEMBER'][name][ifname]:
continue
tagging_mode = conf_facts['VLAN_MEMBER'][name][ifname]['tagging_mode']
port_index = []
if ifname in config_portchannels:
for member in config_portchannels[ifname]['members']:
if conf_facts['port_index_map'][member] in available_ports_idx:
port_index.append(conf_facts['port_index_map'][member])
if port_index:
interface_table[ifname].add(vlan_id)
elif conf_facts['port_index_map'][ifname] in available_ports_idx:
port_index.append(conf_facts['port_index_map'][ifname])
interface_table[ifname].add(vlan_id)
if port_index:
vlan_table[vlan_id].append({
'port_index': port_index,
'tagging_mode': tagging_mode
})
vlan = vlan_table.keys()[0]
vlan_member_count = len(vlan_table[vlan])
total_fdb_entries = min(
TOTAL_FDB_ENTRIES,
(get_crm_resources(duthost, "fdb_entry", "available") -
get_crm_resources(duthost, "fdb_entry", "used")))
dummay_mac_count = int(math.floor(total_fdb_entries / vlan_member_count))
fdb = get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count)
port_list = fdb.keys()
dummy_mac_list = fdb.values()
for loop_time in range(0, loop_times):
port_index_start = (0 + loop_time) % len(port_list)
for (port, dummy_mac_set) in zip(range(len(port_list)),
dummy_mac_list):
port_index = (port_index_start + port) % len(port_list)
for dummy_mac in dummy_mac_set:
send_arp_request(ptfadapter, port_index, dummy_mac, router_mac,
vlan_id)
time.sleep(FDB_POPULATE_SLEEP_TIMEOUT)
pytest_assert(
wait_until(
20, 1, 0, lambda: get_fdb_dynamic_mac_count(duthost) >
vlan_member_count), "FDB Table Add failed")
# Flush dataplane
ptfadapter.dataplane.flush()
fdb_cleanup(duthosts, rand_one_dut_hostname)
# Wait for 10 seconds before starting next loop
time.sleep(10)
def test_verify_status(duthost):
"""Verify procdockerstatsd is active and running
"""
status = duthost.get_service_props('procdockerstatsd')
pytest_assert(status["ActiveState"] == "active" and status["SubState"] == "running", "Procdockerstatsd either not active or not running")
def test_show_platform_syseeprom(duthosts, enum_rand_one_per_hwsku_hostname, dut_vars):
"""
@summary: Verify output of `show platform syseeprom`
"""
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
cmd = " ".join([CMD_SHOW_PLATFORM, "syseeprom"])
logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname))
syseeprom_cmd = duthost.command(cmd)
syseeprom_output = syseeprom_cmd["stdout"]
syseeprom_output_lines = syseeprom_cmd["stdout_lines"]
"""
Gather expected data from a inventory file instead if 'syseeprom_info' is defined in the inventory
# Sample inventory with syseeprom:
str-msn2700-01:
ansible_host: 10.251.0.188
model: MSN2700-CS2FO
serial: MT1234X56789
base_mac: 24:8a:07:12:34:56
syseeprom_info:
"0x21": "MSN2700"
"0x22": "MSN2700-CS2FO"
"0x23": "MT1234X56789"
"0x24": "24:8a:07:12:34:56"
"0x25": "12/07/2016"
"0x26": "0"
"0x28": "x86_64-mlnx_x86-r0"
"0x29": "2016.11-5.1.0008-9600"
"0x2A": "128"
"0x2B": "Mellanox"
"0xFE": "0xFBA1E964"
"""
if 'syseeprom_info' in dut_vars:
expected_syseeprom_info_dict = dut_vars['syseeprom_info']
parsed_syseeprom = {}
# Can't use util.get_fields as the values go beyond the last set of '---' in the hearder line.
regex_int = re.compile(r'([\S\s]+)(0x[A-F0-9]+)\s+([\d]+)\s+([\S\s]*)')
for line in syseeprom_output_lines[6:]:
t1 = regex_int.match(line)
if t1:
parsed_syseeprom[t1.group(2).strip()] = t1.group(4).strip()
for field in expected_syseeprom_info_dict:
pytest_assert(field in parsed_syseeprom, "Expected field '{}' not present in syseeprom on '{}'".format(field, duthost.hostname))
pytest_assert(parsed_syseeprom[field] == expected_syseeprom_info_dict[field],
"System EEPROM info is incorrect - for '{}', rcvd '{}', expected '{}' on '{}'".
format(field, parsed_syseeprom[field], expected_syseeprom_info_dict[field], duthost.hostname))
if duthost.facts["asic_type"] in ["mellanox"]:
expected_fields = [
"Product Name",
"Part Number",
"Serial Number",
"Base MAC Address",
"Manufacture Date",
"Device Version",
"MAC Addresses",
"Manufacturer",
"Vendor Extension",
"ONIE Version",
"CRC-32"]
utility_cmd = "sudo python -c \"import imp; \
m = imp.load_source('eeprom', '/usr/share/sonic/device/{}/plugins/eeprom.py'); \
t = m.board('board', '', '', ''); e = t.read_eeprom(); t.decode_eeprom(e)\"".format(duthost.facts["platform"])
utility_cmd_output = duthost.command(utility_cmd)
for field in expected_fields:
pytest_assert(syseeprom_output.find(field) >= 0, "Expected field '{}' was not found on '{}'".format(field, duthost.hostname))
pytest_assert(utility_cmd_output["stdout"].find(field) >= 0, "Expected field '{}' was not found on '{}'".format(field, duthost.hostname))
for line in utility_cmd_output["stdout_lines"]:
pytest_assert(line in syseeprom_output, "Line '{}' was not found in output on '{}'".format(line, duthost.hostname))
def test_tgen(traffic_config, ixia_api):
config = traffic_config
flow_name = config.flows[0].name
pkt_size = config.flows[0].size.fixed
rate_percent = config.flows[0].rate.value
duration_sec = config.flows[0].duration.seconds.seconds
port_speed = config.layer1[0].speed
words = port_speed.split('_')
pytest_assert(
len(words) == 3 and words[1].isdigit(),
'Fail to get port speed from {}'.format(port_speed))
port_speed_gbps = int(words[1])
""" Apply configuration """
ixia_api.set_state(State(ConfigState(config=config, state='set')))
""" Start traffic """
ixia_api.set_state(State(FlowTransmitState(state='start')))
""" Wait for traffic to finish """
time.sleep(duration_sec)
while True:
rows = ixia_api.get_flow_results(FlowRequest(flow_names=[flow_name]))
if len(rows) == 1 and \
rows[0]['name'] == flow_name and \
rows[0]['transmit'] == 'stopped':
""" Wait for counters to fully propagate """
time.sleep(2)
break
else:
time.sleep(1)
""" Dump per-flow statistics """
rows = ixia_api.get_flow_results(FlowRequest(flow_names=[flow_name]))
""" Stop traffic """
ixia_api.set_state(State(FlowTransmitState(state='stop')))
""" Analyze traffic results """
pytest_assert(
len(rows) == 1 and rows[0]['name'] == flow_name,
'Fail to get results of flow {}'.format(flow_name))
row = rows[0]
rx_frames = row['frames_rx']
tx_frames = row['frames_tx']
pytest_assert(
rx_frames == tx_frames,
'Unexpected packet losses (Tx: {}, Rx: {})'.format(
tx_frames, rx_frames))
tput_bps = port_speed_gbps * 1e9 * rate_percent / 100.0
exp_rx_frames = tput_bps * duration_sec / 8 / pkt_size
deviation_thresh = 0.05
ratio = float(exp_rx_frames) / rx_frames
deviation = abs(ratio - 1)
pytest_assert(
deviation <= deviation_thresh,
'Expected / Actual # of pkts: {} / {}'.format(exp_rx_frames,
rx_frames))
def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo,
fanouthosts, get_loop_times):
"""
Validates that link flap works as expected
"""
duthost = duthosts[rand_one_dut_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory: %s M", start_time_redis_memory)
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logger.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
loop_times = get_loop_times
port_lists = get_port_list(duthost, tbinfo)
candidates = []
for port in port_lists:
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, duthost.hostname, port)
candidates.append((port, fanout, fanout_port))
for loop_time in range(0, loop_times):
watch = False
check_status = False
if loop_time == 0 or loop_time == loop_times - 1:
watch = True
check_status = True
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=watch,
check_status=check_status)
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logger.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory at start: %s M", start_time_redis_memory)
logger.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logger.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logger.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logger.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def __gen_traffic(testbed_config,
port_config_list,
port_id,
duthost,
pause_flow_name,
global_pause,
pause_prio_list,
test_flow_name,
test_flow_prio_list,
test_flow_rate_percent,
bg_flow_name,
bg_flow_prio_list,
bg_flow_rate_percent,
data_flow_dur_sec,
data_flow_delay_sec,
data_pkt_size,
prio_dscp_map):
"""
Generate configurations of flows, including test flows, background flows and
pause storm. Test flows and background flows are also known as data flows.
Args:
testbed_config (obj): testbed L1/L2/L3 configuration
port_config_list (list): list of port configuration
port_id (int): ID of DUT port to test
duthost (Ansible host instance): device under test
pause_flow_name (str): name of pause storm
global_pause (bool): if pause frame is IEEE 802.3X pause
pause_prio_list (list): priorities to pause for pause frames
test_flow_name (str): name of test flows
test_prio_list (list): priorities of test flows
test_flow_rate_percent (int): rate percentage for each test flow
bg_flow_name (str): name of background flows
bg_prio_list (list): priorities of background flows
bg_flow_rate_percent (int): rate percentage for each background flow
data_flow_dur_sec (int): duration of data flows in second
data_flow_delay_sec (int): start delay of data flows in second
data_pkt_size (int): packet size of data flows in byte
prio_dscp_map (dict): Priority vs. DSCP map (key = priority).
Returns:
flows configurations (list): the list should have configurations of
len(test_flow_prio_list) test flow, len(bg_flow_prio_list) background
flows and a pause storm.
"""
result = list()
rx_port_id = port_id
tx_port_id_list, rx_port_id_list = select_ports(port_config_list=port_config_list,
duthost=duthost,
pattern="many to one",
rx_port_id=rx_port_id)
pytest_assert(len(tx_port_id_list) > 0, "Cannot find any TX ports")
tx_port_id = select_tx_port(tx_port_id_list=tx_port_id_list,
rx_port_id=rx_port_id)
pytest_assert(tx_port_id is not None, "Cannot find a suitable TX port")
tx_port_config = next((x for x in port_config_list if x.id == tx_port_id), None)
rx_port_config = next((x for x in port_config_list if x.id == rx_port_id), None)
tx_mac = tx_port_config.mac
if tx_port_config.gateway == rx_port_config.gateway and \
tx_port_config.prefix_len == rx_port_config.prefix_len:
""" If soruce and destination port are in the same subnet """
rx_mac = rx_port_config.mac
else:
rx_mac = tx_port_config.gateway_mac
data_endpoint = PortTxRx(tx_port_name=testbed_config.ports[tx_port_id].name,
rx_port_name=testbed_config.ports[rx_port_id].name)
data_flow_delay_nanosec = sec_to_nanosec(data_flow_delay_sec)
""" Test flows """
for prio in test_flow_prio_list:
eth_hdr = EthernetHeader(src=FieldPattern(tx_mac),
dst=FieldPattern(rx_mac),
pfc_queue=FieldPattern([prio]))
ip_prio = Priority(Dscp(phb=FieldPattern(choice=prio_dscp_map[prio]),
ecn=FieldPattern(choice=Dscp.ECN_CAPABLE_TRANSPORT_1)))
ipv4_hdr = Ipv4Header(src=FieldPattern(tx_port_config.ip),
dst=FieldPattern(rx_port_config.ip),
priority=ip_prio)
test_flow = Flow(
name='{} Prio {}'.format(test_flow_name, prio),
tx_rx=TxRx(data_endpoint),
packet=[Header(choice=eth_hdr), Header(choice=ipv4_hdr)],
size=Size(data_pkt_size),
rate=Rate('line', test_flow_rate_percent),
duration=Duration(FixedSeconds(seconds=data_flow_dur_sec,
delay=data_flow_delay_nanosec,
delay_unit='nanoseconds'))
)
result.append(test_flow)
""" Background flows """
for prio in bg_flow_prio_list:
eth_hdr = EthernetHeader(src=FieldPattern(tx_mac),
dst=FieldPattern(rx_mac),
pfc_queue=FieldPattern([prio]))
ip_prio = Priority(Dscp(phb=FieldPattern(choice=prio_dscp_map[prio]),
ecn=FieldPattern(choice=Dscp.ECN_CAPABLE_TRANSPORT_1)))
ipv4_hdr = Ipv4Header(src=FieldPattern(tx_port_config.ip),
dst=FieldPattern(rx_port_config.ip),
priority=ip_prio)
bg_flow = Flow(
name='{} Prio {}'.format(bg_flow_name, prio),
tx_rx=TxRx(data_endpoint),
packet=[Header(choice=eth_hdr), Header(choice=ipv4_hdr)],
size=Size(data_pkt_size),
rate=Rate('line', bg_flow_rate_percent),
duration=Duration(FixedSeconds(seconds=data_flow_dur_sec,
delay=data_flow_delay_nanosec,
delay_unit='nanoseconds'))
)
result.append(bg_flow)
""" Pause storm """
if global_pause:
pause_pkt = Header(EthernetPause(
dst=FieldPattern(choice='01:80:C2:00:00:01'),
src=FieldPattern(choice='00:00:fa:ce:fa:ce')
))
else:
pause_time = []
for x in range(8):
if x in pause_prio_list:
pause_time.append('ffff')
else:
pause_time.append('0000')
vector = pfc_class_enable_vector(pause_prio_list)
pause_pkt = Header(PfcPause(
dst=FieldPattern(choice='01:80:C2:00:00:01'),
src=FieldPattern(choice='00:00:fa:ce:fa:ce'),
class_enable_vector=FieldPattern(choice=vector),
pause_class_0=FieldPattern(choice=pause_time[0]),
pause_class_1=FieldPattern(choice=pause_time[1]),
pause_class_2=FieldPattern(choice=pause_time[2]),
pause_class_3=FieldPattern(choice=pause_time[3]),
pause_class_4=FieldPattern(choice=pause_time[4]),
pause_class_5=FieldPattern(choice=pause_time[5]),
pause_class_6=FieldPattern(choice=pause_time[6]),
pause_class_7=FieldPattern(choice=pause_time[7]),
))
""" Pause frames are sent from the RX port """
pause_endpoint = PortTxRx(tx_port_name=testbed_config.ports[rx_port_id].name,
rx_port_name=testbed_config.ports[tx_port_id].name)
speed_str = testbed_config.layer1[0].speed
speed_gbps = int(speed_str.split('_')[1])
pause_dur = 65535 * 64 * 8.0 / (speed_gbps * 1e9)
pps = int(2 / pause_dur)
pause_flow = Flow(
name=pause_flow_name,
tx_rx=TxRx(pause_endpoint),
packet=[pause_pkt],
size=Size(64),
rate=Rate('pps', value=pps),
duration=Duration(Continuous(delay=0, delay_unit='nanoseconds'))
)
result.append(pause_flow)
return result
def test_tunnel_memory_leak(
toggle_all_simulator_ports_to_upper_tor,
upper_tor_host, lower_tor_host, ptfhost,
ptfadapter, conn_graph_facts, tbinfo, vmhost,
run_arp_responder
):
"""
Test if there is memory leak for service tunnel_packet_handler.
Send ip packets from standby TOR T1 to Server, standby TOR will
forward the packets to active TOR with tunnel, active TOR will
decapsulate the IPinIP packets, but there is no neighbor for destination
as we remove neighbor before test, tunnel_packet_handler will be
triggered and neighbor will be added. Server will receive the packets.
Check if memory usage is increased after tunnel_packet_handler's
operation. Since tunnel_packet_handler is only triggered by the
first packet, loop the process for all severs to trigger it as much
as possible.
"""
@contextlib.contextmanager
def prepare_services(ptfhost):
"""
Temporarily start arp and icmp service. Make sure to stop garp service,
otherwise, it will add neighbor entry back automatically.
It has to stop garp_service for triggering tunnel_packet_handler.
It has to start arp and icmp service for receiving packets at server side.
"""
ptfhost.shell("supervisorctl stop garp_service")
ptfhost.shell("supervisorctl start arp_responder")
ptfhost.shell("supervisorctl start icmp_responder")
yield
ptfhost.shell("supervisorctl stop arp_responder")
ptfhost.shell("supervisorctl stop icmp_responder")
@contextlib.contextmanager
def remove_neighbor(duthost, server_ip):
"""
Remove ip neighbor before test for triggering tunnel_packet_handler,
restore it after test
"""
flush_neighbor_ct = flush_neighbor(duthost, server_ip, True)
with flush_neighbor_ct:
command = "ip neighbor show %s" % server_ip
output = [_.strip() for _ in duthost.shell(command)["stdout_lines"]]
pytest_assert(not output, "server ip {} isn't flushed in neighbor table.".format(server_ip))
yield
pytest_assert(is_tunnel_packet_handler_running(upper_tor_host),
"tunnel_packet_handler is not running in SWSS conainter.")
ptf_t1_intf = random.choice(get_t1_ptf_ports(lower_tor_host, tbinfo))
all_servers_ips = mux_cable_server_ip(upper_tor_host)
with prepare_services(ptfhost):
# Get the original memeory percent before test
check_memory_leak(upper_tor_host)
for iface, server_ips in all_servers_ips.items():
server_ipv4 = server_ips["server_ipv4"].split("/")[0]
logging.info("Select DUT interface {} and server IP {} to test.".format(iface, server_ipv4))
pkt, exp_pkt = build_packet_to_server(lower_tor_host, ptfadapter, server_ipv4)
rm_neighbor = remove_neighbor(upper_tor_host, server_ipv4)
server_traffic_monitor = ServerTrafficMonitor(
upper_tor_host, ptfhost, vmhost, tbinfo, iface,
conn_graph_facts, exp_pkt, existing=True, is_mocked=False
)
with rm_neighbor, server_traffic_monitor:
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=PACKET_COUNT)
logging.info("Sent {} packets from ptf t1 interface {} on standby TOR {}"
.format(PACKET_COUNT, ptf_t1_intf, lower_tor_host.hostname))
# Check memory usage for every operation, used for debugging if test failed
check_memory_leak(upper_tor_host)
pytest_assert(validate_neighbor_entry_exist(upper_tor_host, server_ipv4),
"The server ip {} doesn't exist in neighbor table on dut {}. \
tunnel_packet_handler isn't triggered.".format(server_ipv4, upper_tor_host.hostname))
pytest_assert(len(server_traffic_monitor.matched_packets) > PACKET_COUNT /2,
"Received {} expected packets for server {}, drop more than 50%."
.format(len(server_traffic_monitor.matched_packets), server_ipv4))
# sleep 10s to wait memory usage stable, check if there is memory leak
time.sleep(10)
check_result = check_memory_leak(upper_tor_host)
pytest_assert(check_result == False, "Test failed because there is memory leak on {}"
.format(upper_tor_host.hostname))
def __verify_results(rows,
duthost,
pause_flow_name,
test_flow_name,
bg_flow_name,
data_flow_dur_sec,
test_flow_rate_percent,
bg_flow_rate_percent,
data_pkt_size,
speed_gbps,
test_flow_pause,
tolerance):
"""
Verify if we get expected experiment results
Args:
rows (list): per-flow statistics
duthost (Ansible host instance): device under test
pause_flow_name (str): name of pause storm
test_flow_name (str): name of test flows
bg_flow_name (str): name of background flows
test_flow_rate_percent (int): rate percentage for each test flow
bg_flow_rate_percent (int): rate percentage for each background flow
data_pkt_size (int): packet size of data flows in byte
speed_gbps (int): link speed in Gbps
test_flow_pause (bool): if test flows are expected to be paused
tolerance (float): maximum allowable deviation
Returns:
N/A
"""
""" All the pause frames should be dropped """
pause_flow_row = next(row for row in rows if row["name"] == pause_flow_name)
tx_frames = pause_flow_row['frames_tx']
rx_frames = pause_flow_row['frames_rx']
pytest_assert(tx_frames > 0 and rx_frames == 0,
'All the pause frames should be dropped')
""" Check background flows """
for row in rows:
if bg_flow_name not in row['name']:
continue
tx_frames = row['frames_tx']
rx_frames = row['frames_rx']
pytest_assert(tx_frames == rx_frames,
'{} should not have any dropped packet'.format(row['name']))
exp_bg_flow_rx_pkts = bg_flow_rate_percent / 100.0 * speed_gbps \
* 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size
deviation = (rx_frames - exp_bg_flow_rx_pkts) / float(exp_bg_flow_rx_pkts)
pytest_assert(abs(deviation) < tolerance,
'{} should receive {} packets (actual {})'.\
format(row['name'], exp_bg_flow_rx_pkts, rx_frames))
""" Check test flows """
for row in rows:
if test_flow_name not in row['name']:
continue
tx_frames = row['frames_tx']
rx_frames = row['frames_rx']
if test_flow_pause:
pytest_assert(tx_frames > 0 and rx_frames == 0,
'{} should be paused'.format(row['name']))
else:
pytest_assert(tx_frames == rx_frames,
'{} should not have any dropped packet'.format(row['name']))
exp_test_flow_rx_pkts = test_flow_rate_percent / 100.0 * speed_gbps \
* 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size
deviation = (rx_frames - exp_test_flow_rx_pkts) / float(exp_test_flow_rx_pkts)
pytest_assert(abs(deviation) < tolerance,
'{} should receive {} packets (actual {})'.\
format(test_flow_name, exp_test_flow_rx_pkts, rx_frames))
if test_flow_pause:
""" In-flight TX bytes of test flows should be held by switch buffer """
tx_frames_total = sum(row['frames_tx'] for row in rows if test_flow_name in row['name'])
tx_bytes_total = tx_frames_total * data_pkt_size
dut_buffer_size = get_egress_lossless_buffer_size(host_ans=duthost)
pytest_assert(tx_bytes_total < dut_buffer_size,
'Total TX bytes {} should be smaller than DUT buffer size {}'.\
format(tx_bytes_total, dut_buffer_size))
def _extract_mirror_payload(self, encapsulated_packet, payload_size):
pytest_assert(len(encapsulated_packet) >= self.OUTER_HEADER_SIZE,
"Incomplete packet, expected at least {} header bytes".format(self.OUTER_HEADER_SIZE))
inner_frame = encapsulated_packet[-payload_size:]
return packet.Ether(inner_frame)
def test_pmon_pcied_stop_and_start_status(check_daemon_status, duthosts,
rand_one_dut_hostname,
data_before_restart):
"""
@summary: This test case is to check the pcied stopped and restarted status
"""
duthost = duthosts[rand_one_dut_hostname]
pre_daemon_status, pre_daemon_pid = duthost.get_pmon_daemon_status(
daemon_name)
logger.info("{} daemon is {} with pid {}".format(daemon_name,
pre_daemon_status,
pre_daemon_pid))
duthost.stop_pmon_daemon(daemon_name, SIG_STOP_SERVICE)
time.sleep(2)
daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
pytest_assert(
daemon_status == expected_stopped_status,
"{} expected stopped status is {} but is {}".format(
daemon_name, expected_stopped_status, daemon_status))
pytest_assert(
daemon_pid == -1,
"{} expected pid is -1 but is {}".format(daemon_name, daemon_pid))
data = collect_data(duthost)
pytest_assert(not data['status'], "DB data is not cleared on daemon stop")
pytest_assert(not data['devices'], "DB data is not cleared on daemon stop")
duthost.start_pmon_daemon(daemon_name)
time.sleep(10)
post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(
daemon_name)
pytest_assert(
post_daemon_status == expected_running_status,
"{} expected restarted status is {} but is {}".format(
daemon_name, expected_running_status, post_daemon_status))
pytest_assert(
post_daemon_pid != -1,
"{} expected pid is -1 but is {}".format(daemon_name, post_daemon_pid))
pytest_assert(
post_daemon_pid > pre_daemon_pid,
"Restarted {} pid should be bigger than {} but it is {}".format(
daemon_name, pre_daemon_pid, post_daemon_pid))
data_after_restart = wait_data(duthost)
pytest_assert(data_after_restart == data_before_restart,
'DB data present before and after restart does not match')
def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
ptfhost, tbinfo, nbrhosts, setup, teardown, all_cfg_facts, established_arp):
"""
Verify tables, databases, and kernel routes are correctly updated when a unsolicited ARP packet changes
the MAC address of learned neighbor.
Test Steps
* Send unsolicited ARP packet into DUT for an IP known by DUT with a different MAC address for the neighbor.
* Change the MAC address of the neighbor VM.
* On local linecard:
* Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC.
* On supervisor card:
* Verify Chassis App DB entry is correct for with the updated MAC address.
* On remote linecards:
* Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC
address
* Verify ASIC DB is updated with new MAC.
* Verify kernel route in remote hosts are still present to inband port.
* Verify that packets can be sent from local and remote linecards to learned address.
Args:
duthosts: The duthosts fixture
enum_rand_one_per_hwsku_frontend_hostname: frontend enumeration fixture
enum_asic_index: asic enumeration fixture
ptfhost: The ptfhost fixure.
tbinfo: The tbinfo fixture
nbrhosts: The nbrhosts fixture.
setup: The setup fixture from this module.
established_arp: The established_arp fixture from this module.
all_cfg_facts: The all_cfg_facts fixture from voq/contest.py
"""
self.ptfhost = ptfhost
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = duthost.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", duthost.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
if eth_cfg == {}:
pytest.skip("Can't run this test without any IP interfaces on ethernet ports")
eth_ports = [intf for intf in eth_cfg]
local_port = random.choice(eth_ports)
logger.info("We will test port: %s on host %s, asic %s", local_port, duthost.hostname, asic.asic_index)
nbr_to_test = []
for neighbor in neighs:
local_ip = neighs[neighbor]['local_addr']
nbr_port = get_port_by_ip(cfg_facts, local_ip)
if local_port == nbr_port:
nbr_to_test.append(neighbor)
logger.info("We will test neighbors: %s", nbr_to_test)
for neighbor in nbr_to_test:
nbrinfo = get_neighbor_info(neighbor, nbrhosts)
tb_port = get_ptf_port(duthosts,
all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts'],
tbinfo, duthost, local_port)[0]
original_mac = nbrinfo['mac']
logger.info("*" * 60)
logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port)
logger.info("%s port %s is on ptf port: %s", duthost.hostname, local_port, tb_port)
logger.info("-" * 60)
sonic_ping(asic, neighbor)
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
try:
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC)
self.send_grat_pkt(NEW_MAC, neighbor, int(tb_port))
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=False),
"MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor))
try:
sonic_ping(asic, neighbor)
except AssertionError:
logging.info("No initial response from ping, begin poll to see if ARP table responds.")
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=True),
"MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
ping_all_neighbors(duthosts, all_cfg_facts, [neighbor])
finally:
logger.info("Will Restore ethernet mac on neighbor: %s, port %s, vm %s", neighbor,
nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac)
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(
wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
ping_all_neighbors(duthosts, all_cfg_facts, [neighbor])
def toggle_all_simulator_ports_to_random_side(duthosts, mux_server_url, tbinfo):
"""
A function level fixture to toggle all ports to a random side.
"""
def _get_mux_status(duthost):
cmd = 'show mux status --json'
return json.loads(duthost.shell(cmd)['stdout'])
def _check_mux_status_consistency():
"""Ensure mux status is consistent between the ToRs and mux simulator."""
upper_tor_mux_status = _get_mux_status(upper_tor_host)
lower_tor_mux_status = _get_mux_status(lower_tor_host)
simulator_mux_status = _get(mux_server_url)
if not upper_tor_mux_status:
logging.warn("Failed to retrieve mux status from the upper tor")
return False
if not lower_tor_mux_status:
logging.warn("Failed to retrieve mux status from the lower tor")
return False
if not simulator_mux_status:
logging.warn("Failed to retrieve mux status from the mux simulator")
return False
if not set(upper_tor_mux_status.keys()) == set(lower_tor_mux_status.keys()):
logging.warn("Ports mismatch between the upper tor and lower tor")
return False
# get mapping from port indices to mux status
simulator_port_mux_status = {int(k.split('-')[-1]):v for k,v in simulator_mux_status.items()}
for intf in upper_tor_mux_status['MUX_CABLE']:
intf_index = port_indices[intf]
if intf_index not in simulator_port_mux_status:
logging.warn("No mux status for interface %s from mux simulator", intf)
return False
simulator_status = simulator_port_mux_status[intf_index]
upper_tor_status = upper_tor_mux_status['MUX_CABLE'][intf]['STATUS']
lower_tor_status = lower_tor_mux_status['MUX_CABLE'][intf]['STATUS']
if upper_tor_status == 'active' and lower_tor_status == 'standby' and simulator_status['active_side'] == 'upper_tor':
continue
if upper_tor_status == 'standby' and lower_tor_status == 'active' and simulator_status['active_side'] == 'lower_tor':
continue
logging.warn(
"For interface %s, upper tor mux status: %s, lower tor mux status: %s, simulator status: %s",
intf, upper_tor_status, lower_tor_status, simulator_status
)
logging.warn("Inconsistent mux status for interface %s", intf)
return False
return True
if 'dualtor' not in tbinfo['topo']['name']:
return
_toggle_all_simulator_ports(mux_server_url, RANDOM, tbinfo)
upper_tor_host, lower_tor_host = duthosts[0], duthosts[1]
mg_facts = upper_tor_host.get_extended_minigraph_facts(tbinfo)
port_indices = mg_facts['minigraph_port_indices']
pytest_assert(
utilities.wait_until(30, 5, 0, _check_mux_status_consistency),
"Mux status is inconsistent between the DUTs and mux simulator after toggle"
)
def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly updated when the MAC address of a neighbor changes
and is updated via request/reply exchange.
Test Steps
* Change the MAC address on a remote host that is already present in the ARP table.
* Without clearing the entry in the DUT, allow the existing entry to time out and the new reply to have the new MAC
address.
* On local linecard:
* Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC.
* On supervisor card:
* Verify Chassis App DB entry is correct for with the updated MAC address.
* On remote linecards:
* Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC
address
* Verify ASIC DB is updated with new MAC.
* Verify kernel route in remote hosts are still present to inband port.
* Verify that packets can be sent from local and remote linecards to the learned address.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish arp on all nodes
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
if eth_cfg == {}:
pytest.skip("Can't run this test without any IP interfaces on ethernet ports")
eth_ports = [intf for intf in eth_cfg]
local_port = random.choice(eth_ports)
logger.info("We will test port: %s on host %s, asic %s", local_port, per_host.hostname, asic.asic_index)
nbr_to_test = []
for neighbor in neighs:
local_ip = neighs[neighbor]['local_addr']
nbr_port = get_port_by_ip(cfg_facts, local_ip)
if local_port == nbr_port:
nbr_to_test.append(neighbor)
logger.info("We will test neighbors: %s", nbr_to_test)
nbrinfo = get_neighbor_info(nbr_to_test[0], nbrhosts)
original_mac = nbrinfo['mac']
for neighbor in nbr_to_test:
# Check neighbor on local linecard
logger.info("*" * 60)
logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
try:
logger.info("Changing ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False),
"MAC {} didn't change in ARP table".format(NEW_MAC))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC),
"MAC {} didn't change in ARP table".format(NEW_MAC))
logger.info("Verify neighbor after mac change: %s, port %s", neighbor, local_port)
check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
logger.info("Ping neighbors: %s from all line cards", nbr_to_test)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
finally:
logger.info("-" * 60)
logger.info("Will Restore ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(
wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
def _reset_simulator_port(interface_name=None):
logger.warn("Resetting simulator ports {}".format('all' if interface_name is None else interface_name))
server_url = url(interface_name=interface_name, action=RESET)
pytest_assert(_post(server_url, {}))
def generate_and_verify_decap_traffic(duthost,
ptfadapter,
src_port,
dst_port,
ip_src,
ip_dst,
tr_type,
ip_tunnel=None):
"""
Send encapsulated packet from PTF to DUT and
verify that DUT sends/doesn't send TCP/UDP packet to PTF.
Args:
duthost: DUT host object
ptfadapter: PTF adapter
src_port: Source port of PTF
dst_port: Destination port of PTF
ip_src: Source IP address of PTF
ip_dst: Destination IP address of PTF
tr_type: Type of traffic (TCP or UDP)
ip_tunnel: Tunnel IP address of DUT
"""
router_mac = duthost.facts['router_mac']
src_port_number = int(get_port_number(src_port))
dst_port_number = int(get_port_number(dst_port))
ip_src = ip_src.split('/')[0]
ip_dst = ip_dst.split('/')[0]
ip_tunnel = ip_tunnel.split('/')[0]
# Define encapsulated packet
pkt = create_packet(eth_dst=router_mac,
eth_src=ptfadapter.dataplane.get_mac(
0, src_port_number),
ip_src=ip_src,
ip_dst=ip_dst,
ip_tunnel=ip_tunnel,
vlan_vid=int(src_port.split('.')[1]),
dl_vlan_enable=True,
tr_type=tr_type,
ttl=64)
# Build expected packet
inner_packet = pkt[packet.IP].payload[packet.IP].copy()
exp_pkt = Ether(src=router_mac,
dst=ptfadapter.dataplane.get_mac(
0, dst_port_number)) / Dot1Q(
vlan=int(dst_port.split('.')[1])) / inner_packet
exp_pkt['IP'].ttl -= 1
update_dut_arp_table(duthost, ip_dst)
ptfadapter.dataplane.flush()
testutils.send_packet(ptfadapter, src_port_number, pkt)
pkt_filter = FilterPktBuffer(ptfadapter=ptfadapter,
exp_pkt=exp_pkt,
dst_port_number=dst_port_number,
match_fields=[("802.1Q", "vlan"),
("Ethernet", "src"),
("Ethernet", "dst"),
("IP", "src"), ("IP", "dst")],
ignore_fields=[])
pkt_in_buffer = pkt_filter.filter_pkt_in_buffer()
pytest_assert(pkt_in_buffer is True,
"Expected packet not available:\n{}".format(pkt_in_buffer))