def start_fuz(tc):
ret = copy_fuz(tc)
if ret != api.types.status.SUCCESS:
return api.types.status.FAILURE
tc.serverCmds = []
tc.clientCmds = []
tc.cmd_descr = []
serverReq = None
clientReq = None
serverReq = api.Trigger_CreateExecuteCommandsRequest(serial = False)
clientReq = api.Trigger_CreateExecuteCommandsRequest(serial = False)
# ping test above sets the workload pairs to remote only
# setting wl_pairs as per arg selected in testbundle
workload_pairs = []
if tc.args.type == 'local_only':
api.Logger.info("local_only test")
workload_pairs = api.GetLocalWorkloadPairs()
elif tc.args.type == 'both':
api.Logger.info(" both local and remote test")
workload_pairs = api.GetLocalWorkloadPairs()
workload_pairs.extend(api.GetRemoteWorkloadPairs())
else:
api.Logger.info("remote_only test")
workload_pairs = api.GetRemoteWorkloadPairs()
wl_under_move = []
for wl_info in tc.move_info:
wl_under_move.append(wl_info.wl)
'''
tc.workload_pairs updated in ping test above
resetting that to limit fuz tests to vm under move
'''
tc.workload_pairs = []
for pairs in workload_pairs:
if pairs[0] in wl_under_move or pairs[1] in wl_under_move:
api.Logger.info("Adding %s and %s for fuz test" %(pairs[0].workload_name, pairs[1].workload_name))
tc.workload_pairs.append(pairs)
for idx, pairs in enumerate(tc.workload_pairs):
client = pairs[0]
server = pairs[1]
cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address, client.workload_name, client.ip_address)
tc.cmd_descr.append(cmd_descr)
num_sessions = int(getattr(tc.args, "num_sessions", 1))
serverCmd = None
clientCmd = None
port = api.AllocateTcpPort()
api.Logger.info("Starting Fuz test from %s num-sessions %d Port %d" % (cmd_descr, num_sessions, port))
serverCmd = tc.fuz_exec[server.workload_name] + " -port " + str(port)
clientCmd = tc.fuz_exec[client.workload_name] + " -conns " + str(num_sessions) + " -duration " + str(__fuz_run_time) + " -attempts 1 -read-timeout 20 -talk " + server.ip_address + ":" + str(port)
tc.serverCmds.append(serverCmd)
tc.clientCmds.append(clientCmd)
api.Trigger_AddCommand(serverReq, server.node_name, server.workload_name,
serverCmd, background = True)
api.Trigger_AddCommand(clientReq, client.node_name, client.workload_name,
clientCmd, background = True)
tc.server_resp = api.Trigger(serverReq)
#Sleep for some time as bg may not have been started.
time.sleep(5)
tc.fuz_client_resp = api.Trigger(clientReq)
return api.types.status.SUCCESS
def CheckRolloutStatus(tc):
retries = 0
start_ts = datetime.now()
result = api.types.status.FAILURE
status_found = False
while retries < 100:
api.Logger.info("------Issuing Rollout get %s retry------"%retries)
misc_utils.Sleep(2)
retries += 1
# get rollout status
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
api.Logger.info("Sending rollout status get request: %s"%(cmd))
tc.resp = api.Trigger(req)
try:
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
except Exception as e:
api.Logger.error(f"Exception occured in sending rollout status get.{e}")
continue
for cmd in tc.resp.commands:
if cmd.exit_code != 0:
api.Logger.info("Rollout status get request returned failure")
continue
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
status_found = True
if not item['Op'] == 4:
api.Logger.info("opcode is bad for %s"%cmd.node_name)
result = api.types.status.FAILURE
if "fail" in tc.iterators.option:
if not item['opstatus'] == 'failure':
api.Logger.info("opstatus is bad for %s"%cmd.node_name)
result = api.types.status.FAILURE
if tc.iterators.option not in item['Message']:
api.Logger.info("message is bad")
result = api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus(%s) is bad for %s"%(item['opstatus'], cmd.node_name))
result = api.types.status.FAILURE
else:
api.Logger.info("Rollout status is SUCCESS for %s"%cmd.node_name)
result = api.types.status.SUCCESS
except Exception as e:
api.Logger.error("resp: ", json.dumps(resp, indent=1))
#api.Logger.error(f"Exception occured in parsing response: {e}")
result = api.types.status.FAILURE
continue
if status_found:
break
end_ts = datetime.now()
# find time elapsed in retrieving rollout status and adjust the wait time for traffic test.
timedelta = end_ts - start_ts
time_elapsed = timedelta.days * 24 * 3600 + timedelta.seconds
if time_elapsed < 100:
time_elapsed = 100
misc_utils.Sleep(time_elapsed)
tc.sleep = (tc.sleep - time_elapsed) if (tc.sleep > time_elapsed) else 10
return result
def Trigger(tc):
tc.cmd_cookies = []
tc.cmd_cookies1 = []
tc.cmd_cookies2 = []
nodes = api.GetWorkloadNodeHostnames()
push_node_0 = [nodes[0]]
push_node_1 = [nodes[1]]
encrypt_objects = netagent_cfg_api.QueryConfigs(kind='IPSecSAEncrypt')
decrypt_objects = netagent_cfg_api.QueryConfigs(kind='IPSecSADecrypt')
policy_objects = netagent_cfg_api.QueryConfigs(kind='IPSecPolicy')
# Configure IPsec on Node 1
if api.IsNaplesNode(nodes[0]):
if len(encrypt_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_encryption_node1.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_encryption_node1.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_0,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-encryption objects to node %s" %
nodes[0])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
return api.types.status.FAILURE
if len(decrypt_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_decryption_node1.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_decryption_node1.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_0,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-encryption objects to node %s" %
nodes[0])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
return api.types.status.FAILURE
if len(policy_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_policies_node1.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_policies_node1.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_0,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-policy objects to node %s" %
nodes[0])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
return api.types.status.FAILURE
else:
workloads = api.GetWorkloads(nodes[0])
w1 = workloads[0]
req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddCommand(req1, w1.node_name, w1.workload_name,
"sudo ip xfrm state flush")
api.Trigger_AddCommand(req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy flush")
for port, spi, aead in zip(tc.args.ports_list, tc.args.spi_list,
tc.args.aead_list):
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto tcp dport %s dir in tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto tcp dport %s dir fwd tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto tcp sport %s dir out tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm state add src 192.168.100.103 dst 192.168.100.101 proto esp spi %s mode tunnel aead 'rfc4106(gcm(aes))' %s 128 sel src 192.168.100.103/32 dst 192.168.100.101/32"
% (spi, aead))
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm state add src 192.168.100.101 dst 192.168.100.103 proto esp spi %s mode tunnel aead 'rfc4106(gcm(aes))' %s 128 sel src 192.168.100.101/32 dst 192.168.100.103/32"
% (spi, aead))
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto udp dport %s dir in tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto udp dport %s dir fwd tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto udp sport %s dir out tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(req1, w1.node_name, w1.workload_name,
"sudo ip xfrm state list")
api.Trigger_AddCommand(req1, w1.node_name, w1.workload_name,
"sudo ip xfrm policy list")
trig_resp1 = api.Trigger(req1)
term_resp1 = api.Trigger_TerminateAllCommands(trig_resp1)
# Configure IPsec on Node 2
if api.IsNaplesNode(nodes[1]):
if len(encrypt_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_encryption_node2.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_encryption_node2.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_1,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-encryption objects to node %s" %
nodes[1])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
#return api.types.status.FAILURE
if len(decrypt_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_decryption_node2.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_decryption_node2.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_1,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-encryption objects to node %s" %
nodes[1])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
#return api.types.status.FAILURE
if len(policy_objects) == 0:
newObjects = netagent_cfg_api.AddOneConfig(
api.GetTopologyDirectory() +
"/ipsec/ipsec_policies_node2.json")
if len(newObjects) == 0:
api.Logger.error(
"Adding new objects to store failed for ipsec_policies_node2.json"
)
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(newObjects,
node_names=push_node_1,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error(
"Unable to push ipsec-encryption objects to node %s" %
nodes[1])
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(newObjects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
#return api.types.status.FAILURE
else:
workloads = api.GetWorkloads(nodes[1])
w2 = workloads[0]
req2 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddCommand(req2, w2.node_name, w2.workload_name,
"sudo ip xfrm state flush")
api.Trigger_AddCommand(req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy flush")
for port, spi, aead in zip(tc.args.ports_list, tc.args.spi_list,
tc.args.aead_list):
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto tcp dport %s dir in tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto tcp dport %s dir fwd tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto tcp sport %s dir out tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm state add src 192.168.100.101 dst 192.168.100.103 proto esp spi %s mode tunnel aead 'rfc4106(gcm(aes))' %s 128 sel src 192.168.100.101/32 dst 192.168.100.103/32"
% (spi, aead))
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm state add src 192.168.100.103 dst 192.168.100.101 proto esp spi %s mode tunnel aead 'rfc4106(gcm(aes))' %s 128 sel src 192.168.100.103/32 dst 192.168.100.101/32"
% (spi, aead))
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto udp dport %s dir in tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.101/32 dst 192.168.100.103/32 proto udp dport %s dir fwd tmpl src 192.168.100.101 dst 192.168.100.103 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(
req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy add src 192.168.100.103/32 dst 192.168.100.101/32 proto udp sport %s dir out tmpl src 192.168.100.103 dst 192.168.100.101 proto esp mode tunnel"
% port)
api.Trigger_AddCommand(req2, w2.node_name, w2.workload_name,
"sudo ip xfrm state list")
api.Trigger_AddCommand(req2, w2.node_name, w2.workload_name,
"sudo ip xfrm policy list")
trig_resp2 = api.Trigger(req2)
term_resp2 = api.Trigger_TerminateAllCommands(trig_resp2)
workloads = api.GetWorkloads(nodes[0])
w1 = workloads[0]
workloads = api.GetWorkloads(nodes[1])
w2 = workloads[0]
bypass_test = 0
if w1.IsNaples() and w2.IsNaples():
api.Logger.info(
"Both workloads are Naples, %s is iperf client, %s is iperf server, bypassing test"
% (w1.node_name, w2.node_name))
iperf_client_wl = w1
iperf_server_wl = w2
bypass_test = 1
elif w1.IsNaples():
api.Logger.info("%s is Naples and iperf client, %s is iperf server" %
(w1.node_name, w2.node_name))
iperf_client_wl = w1
iperf_server_wl = w2
elif w2.IsNaples():
api.Logger.info("%s is Naples and iperf client, %s is iperf server" %
(w2.node_name, w1.node_name))
iperf_client_wl = w2
iperf_server_wl = w1
req3 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "Set rcv socket buffer size on %s" % (w1.workload_name)
api.Trigger_AddCommand(
req3, w1.node_name, w1.workload_name,
"sysctl -w net.ipv4.tcp_rmem='4096 2147483647 2147483647'")
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "Set rcv socket buffer size on %s" % (w2.workload_name)
api.Trigger_AddCommand(
req3, w2.node_name, w2.workload_name,
"sysctl -w net.ipv4.tcp_rmem='4096 2147483647 2147483647'")
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "Setting MTU to smaller value on %s" % (
iperf_client_wl.workload_name)
api.Trigger_AddCommand(req3, iperf_client_wl.node_name,
iperf_client_wl.workload_name,
"ifconfig %s mtu 1048" % iperf_client_wl.interface)
tc.cmd_cookies.append(cmd_cookie)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s) on %s" %\
(iperf_server_wl.workload_name, iperf_server_wl.ip_address, iperf_client_wl.workload_name, iperf_client_wl.ip_address, tc.iterators.protocol)
api.Logger.info("Starting Iperf test over IPSec from %s" % (tc.cmd_descr))
if bypass_test == 0:
for port in tc.args.ports_list:
cmd_cookie = "Running iperf server on %s port %s" % (
iperf_server_wl.workload_name, port)
api.Trigger_AddCommand(req3,
iperf_server_wl.node_name,
iperf_server_wl.workload_name,
"iperf -s -p %s" % (port),
background=True)
tc.cmd_cookies.append(cmd_cookie)
req4 = api.Trigger_CreateExecuteCommandsRequest(serial=False)
if bypass_test == 0:
cmd_cookie = "Brief Sleep"
api.Trigger_AddCommand(req4, iperf_client_wl.node_name,
iperf_client_wl.workload_name, "sleep 1")
tc.cmd_cookies.append(cmd_cookie)
for port in tc.args.ports_list:
cmd_cookie = "Running iperf client on %s port %s" % (
iperf_client_wl.workload_name, port)
if tc.iterators.protocol == "tcp":
api.Trigger_AddCommand(
req4, iperf_client_wl.node_name,
iperf_client_wl.workload_name, "iperf -c %s -p %s -M %s" %
(iperf_server_wl.ip_address, port, tc.iterators.pktsize))
else:
api.Trigger_AddCommand(
req4, iperf_client_wl.node_name,
iperf_client_wl.workload_name,
"iperf --udp -c %s -p %s -M %s" %
(iperf_server_wl.ip_address, port, tc.iterators.pktsize))
tc.cmd_cookies.append(cmd_cookie)
req5 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if w1.IsNaples():
cmd_cookie = "IPSec state on %s AFTER running iperf traffic" % (
w1.node_name)
api.Trigger_AddNaplesCommand(
req5, w1.node_name, "/nic/bin/halctl show ipsec-global-stats")
tc.cmd_cookies.append(cmd_cookie)
else:
cmd_cookie = "IPSec state on %s AFTER running iperf traffic" % (
w1.node_name)
api.Trigger_AddCommand(req5, w1.node_name, w1.workload_name,
"sudo ip xfrm policy show")
tc.cmd_cookies.append(cmd_cookie)
if w2.IsNaples():
cmd_cookie = "IPSec state on %s AFTER running iperf traffic" % (
w2.node_name)
api.Trigger_AddNaplesCommand(
req5, w2.node_name, "/nic/bin/halctl show ipsec-global-stats")
tc.cmd_cookies.append(cmd_cookie)
else:
cmd_cookie = "IPSec state on %s AFTER running iperf traffic" % (
w2.node_name)
api.Trigger_AddCommand(req5, w2.node_name, w2.workload_name,
"sudo ip xfrm policy show")
tc.cmd_cookies.append(cmd_cookie)
trig_resp3 = api.Trigger(req3)
trig_resp4 = api.Trigger(req4)
trig_resp5 = api.Trigger(req5)
term_resp3 = api.Trigger_TerminateAllCommands(trig_resp3)
term_resp4 = api.Trigger_TerminateAllCommands(trig_resp4)
term_resp5 = api.Trigger_TerminateAllCommands(trig_resp5)
agg_resp4 = api.Trigger_AggregateCommandsResponse(trig_resp4, term_resp4)
tc.resp = agg_resp4
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cmd_cookies1 = []
#Step 0: Update the timeouti & connection-tracking in the config object
if not tc.skip_security_prof:
update_timeout("icmp-timeout", tc.iterators.timeout)
w1, w2 = tc.workload_pairs[0]
cmd_cookie = "%s(%s) --> %s(%s)" %\
(w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address)
api.Logger.info("Starting Ping test from %s" % (cmd_cookie))
cmd_cookie = "halctl clear session"
api.Trigger_AddNaplesCommand(req, tc.old_node,
"/nic/bin/halctl clear session")
tc.cmd_cookies1.append(cmd_cookie)
if not tc.skip_security_prof:
timeout = timetoseconds(tc.iterators.timeout) + GRACE_TIME
else:
timeout = str(DEFAULT_ICMP_TIMEOUT + GRACE_TIME)
cmd_cookie = "ping"
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
"ping -c 180 %s" % w2.ip_address,
background=True)
tc.cmd_cookies1.append(cmd_cookie)
cmd_cookie = "Before move show session ICMP"
api.Trigger_AddNaplesCommand(req, tc.old_node,
"/nic/bin/halctl show session | grep ICMP")
tc.cmd_cookies1.append(cmd_cookie)
tc.resp1 = api.Trigger(req)
# vm_utils.do_vmotion(tc, tc.wl, tc.new_node)
vm_utils.do_vmotion(tc, True)
#import pdb; pdb.set_trace()
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cmd_cookies = []
cmd_cookie = "After move show session ICMP"
api.Trigger_AddNaplesCommand(req, tc.wl.node_name,
"/nic/bin/halctl show session | grep ICMP")
tc.cmd_cookies.append(cmd_cookie)
#Get it from the config
cmd_cookie = "sleep"
api.Trigger_AddNaplesCommand(req,
tc.wl.node_name,
"sleep 180",
timeout=210)
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "After aging show session"
api.Trigger_AddNaplesCommand(req, tc.wl.node_name,
"/nic/bin/halctl show session | grep ICMP")
tc.cmd_cookies.append(cmd_cookie)
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
#==============================================================
# trigger the commands
#==============================================================
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
# Populate bw lookup table - manual entry to speed up development
bw_dict = {}
bw_dict[(1, 4096)] = 10
bw_dict[(1, 8192)] = 10
bw_dict[(1, 65536)] = 50
bw_dict[(2, 4000)] = 10
bw_dict[(2, 4096)] = 10
bw_dict[(2, 8192)] = 10
bw_dict[(2, 16384)] = 10
bw_dict[(2, 32768)] = 30
bw_dict[(2, 65536)] = 50
bw_dict[(2, 8000)] = 10
bw_dict[(2, 16000)] = 10
bw_dict[(2, 32000)] = 30
bw_dict[(2, 64000)] = 50
bw_dict[(3, 4095)] = 5
bw_dict[(3, 3072)] = 5
bw_dict[(3, 3000)] = 5
bw_dict[(3, 12288)] = 10
bw_dict[(3, 24576)] = 20
bw_dict[(3, 12000)] = 10
bw_dict[(3, 24000)] = 20
bw_dict[(4, 4000)] = 5
bw_dict[(4, 4096)] = 5
bw_dict[(4, 8192)] = 10
bw_dict[(4, 16384)] = 10
bw_dict[(4, 32768)] = 30
bw_dict[(4, 65536)] = 50
bw_dict[(4, 16000)] = 10
bw_dict[(4, 32000)] = 30
bw_dict[(4, 64000)] = 50
bw_dict[(5, 20480)] = 20
bw_dict[(5, 20000)] = 10
bw_dict[(5, 10000)] = 5
bw_dict[(6, 12288)] = 10
bw_dict[(6, 24576)] = 20
bw_dict[(6, 24000)] = 20
bw_dict[(7, 28672)] = 20
bw_dict[(7, 28000)] = 30
bw_dict[(7, 7700)] = 4
bw_dict[(8, 16384)] = 5
bw_dict[(8, 32768)] = 10
bw_dict[(8, 65536)] = 10
bw_dict[(8, 32000)] = 10
bw_dict[(8, 64000)] = 10
#==============================================================
# init cmd options
#==============================================================
iter_opt = ' -n 10 '
misc_opt = ' -F --report_gbits '
cm_opt = ''
enable_dcqcn = False
transport_opt = ''
msg_size = 65536
size_opt = ' -a '
mtu_opt = ' -m 4096 '
qp_opt = ''
numsges_opt = ''
bidir_opt = ''
rxdepth_opt = ''
txdepth_opt = ''
atomic_opt = ''
tc.client_bkg = False
s_port = 12340
e_port = s_port + 1
server_idx = 0
client_idx = 1
bkg_timeout = 130
sq_drain_opt = ''
async_event_stats_opt = ''
bw_opt = ''
port_flap = False
tc.tcpdump = False
#==============================================================
# update non-default cmd options
#==============================================================
# if use both duration '-D' and count '-n', count will take precedence
if hasattr(tc.iterators, 'duration'):
iter_opt = ' -D {} '.format(tc.iterators.duration)
# For scale tests, we noticed all 8 threads not started early,
# so need to give extra timeout
bkg_timeout = tc.iterators.duration + 60
if hasattr(tc.iterators, 'count'):
iter_opt = ' -n {} '.format(tc.iterators.count)
if getattr(tc.iterators, 'rdma_cm', None) == 'yes':
cm_opt = ' -R '
if getattr(tc.iterators, 'transport', None) == 'UD':
transport_opt = ' -c UD '
if hasattr(tc.iterators, 'size'):
msg_size = int(tc.iterators.size)
size_opt = ' -s {} '.format(msg_size)
if hasattr(tc.iterators, 'mtu'):
mtu_opt = ' -m {} '.format(tc.iterators.mtu)
numsges = getattr(tc.iterators, 'numsges', 1)
if numsges > 1:
numsges_opt = ' -W {} '.format(numsges)
num_qp = getattr(tc.iterators, 'num_qp', 1)
if num_qp > 1:
qp_opt = ' -q {} '.format(num_qp)
num_threads = getattr(tc.iterators, 'threads', 1)
if num_threads > 1:
tc.client_bkg = True
e_port = s_port + tc.iterators.threads
if getattr(tc.iterators, 'server', None) == 'no':
server_idx = 1
client_idx = 0
if getattr(tc.iterators, 'bidir', None) == 'yes':
bidir_opt = ' -b '
if hasattr(tc.iterators, 'rxdepth'):
rxdepth_opt = ' -r {} '.format(tc.iterators.rxdepth)
if hasattr(tc.iterators, 'txdepth'):
txdepth_opt = ' -t {} '.format(tc.iterators.txdepth)
if getattr(tc.iterators, 'cmp_swp', None) == 'yes':
atomic_opt = ' -A CMP_AND_SWAP '
if getattr(tc.iterators, 'enable_dcqcn', None) == 'yes':
enable_dcqcn = True
if getattr(tc.iterators, 'sq_drain', None) == 'yes':
sq_drain_opt = ' --sq-drain '
if getattr(tc.iterators, 'async_event_stats', None) == 'yes':
async_event_stats_opt = ' --report-async-ev-stats '
if getattr(tc.iterators, 'check_bw', None) == 'yes' and \
num_qp == 1 and \
(numsges, msg_size) in bw_dict:
bw_opt = ' -w {} '.format(
math.ceil(bw_dict[(numsges, msg_size)] / num_threads))
if getattr(tc.iterators, 'port_flap', None) == 'true' and \
hasattr(tc.iterators, 'duration'):
port_flap = True
tc.client_bkg = True
if getattr(tc.iterators, 'tcpdump', None) == 'yes' and \
not hasattr(tc.iterators, 'duration'):
tc.tcpdump = True
iter_opt = ' -n 5 '
#==============================================================
# run the cmds
#==============================================================
w1 = tc.w[server_idx]
w2 = tc.w[client_idx]
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address)
api.Logger.info("Starting %s test from %s" %
(tc.iterators.command, tc.cmd_descr))
# Enable rdma sniffer and start tcpdump on Naples Hosts
if tc.tcpdump == True:
for w in [w1, w2]:
if not w.IsNaples():
continue
tcpdump_intf = w.interface.split('.')[
0] # Get the parent interface
tcpdump_cmd = "sudo tcpdump -l --immediate-mode -i {} -XXX udp dst port 4791 -w rdma_capture.pcap &".format(
tcpdump_intf)
if tc.os == host.OS_TYPE_BSD:
sniffer_cmd = 'sysctl dev.' + host.GetNaplesSysctl(
w.interface) + '.rdma_sniffer=1'
elif tc.os == host.OS_TYPE_LINUX:
sniffer_cmd = 'sudo ethtool --set-priv-flags ' + tcpdump_intf + ' rdma-sniffer on'
else:
continue
api.Trigger_AddCommand(req, w.node_name, w.workload_name,
sniffer_cmd)
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
tcpdump_cmd,
background=True)
if enable_dcqcn == True:
for w in [w1, w2]:
if not w.IsNaples():
continue
if tc.os == host.OS_TYPE_BSD:
cmd = 'sysctl sys.class.infiniband.' + host.GetNaplesSysClassSysctl(
w.interface) + '.dcqcn.match_default="1"'
elif tc.os == host.OS_TYPE_LINUX:
cmd = 'echo 1 > /sys/class/infiniband/' + host.GetNaplesSysClassSysctl(
w.interface) + '/dcqcn/match_default'
else:
continue
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
cmd,
timeout=120)
#==============================================================
# cmd for server
#==============================================================
for p in range(s_port, e_port):
port_opt = ' -p {} '.format(p)
dev_opt = ' -d {} '.format(tc.devices[server_idx])
gid_opt = ' -x {} '.format(tc.gid[server_idx])
cmd = tc.iterators.command
cmd += dev_opt + iter_opt + gid_opt
cmd += size_opt + mtu_opt + qp_opt
cmd += cm_opt + transport_opt + misc_opt + port_opt + bidir_opt + rxdepth_opt + txdepth_opt + atomic_opt + bw_opt
# add numsges_opt only for Naples
if w1.IsNaples():
cmd += numsges_opt
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
tc.ib_prefix[server_idx] + cmd,
background=True,
timeout=120)
# On Naples-Mellanox setups, with Mellanox as server, it takes a few seconds before the server
# starts listening. So sleep for a few seconds before trying to start the client
cmd = 'sleep 2'
api.Trigger_AddCommand(req, w2.node_name, w2.workload_name, cmd)
#==============================================================
# cmd for client
#==============================================================
for p in range(s_port, e_port):
port_opt = ' -p {} '.format(p)
dev_opt = ' -d {} '.format(tc.devices[client_idx])
gid_opt = ' -x {} '.format(tc.gid[client_idx])
cmd = tc.iterators.command
cmd += dev_opt + iter_opt + gid_opt
cmd += size_opt + mtu_opt + qp_opt
cmd += cm_opt + transport_opt + misc_opt + port_opt + bidir_opt + rxdepth_opt + txdepth_opt + atomic_opt
# add numsges_opt only for Naples
if w2.IsNaples():
cmd += numsges_opt + sq_drain_opt + async_event_stats_opt
# append server's ip_address
cmd += w1.ip_address
api.Trigger_AddCommand(
req,
w2.node_name,
w2.workload_name,
tc.ib_prefix[client_idx] + cmd,
background=tc.client_bkg,
timeout=125) #5 secs more than def test timeout=120
#Do the port flap only for duration tests
if hasattr(tc.iterators, 'duration') and port_flap == True:
num_flaps = int(getattr(tc.iterators, 'duration')) // 20
num_flaps = num_flaps - 2 #Reduce the number of flaps so that we don't flap during connection close
export_path_cmd = "export PATH=$PATH:/platform/bin:/nic/bin:/platform/tools:/nic/tools"
export_ld_path_cmd = "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib:/nic/lib"
port_down_cmd = "/nic/bin/halctl debug port --port 1 --admin-state down"
port_up_cmd = "/nic/bin/halctl debug port --port 1 --admin-state up"
#Sleep for 10 to make sure that we don't flap during connection create
cmd = 'sleep 10'
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
cmd,
timeout=20)
for i in range(num_flaps):
api.Trigger_AddNaplesCommand(req, w1.node_name, export_path_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, export_path_cmd)
api.Trigger_AddNaplesCommand(req, w1.node_name, export_ld_path_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, export_ld_path_cmd)
api.Trigger_AddNaplesCommand(req, w1.node_name, port_down_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, port_down_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, "sleep 1")
api.Trigger_AddNaplesCommand(req, w1.node_name, port_up_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, port_up_cmd)
api.Trigger_AddNaplesCommand(req, w2.node_name, "sleep 20")
#Sleep to let the tests complete before Terminating
cmd = 'sleep 30'
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
cmd,
timeout=40)
if tc.client_bkg and port_flap == False:
# since the client is running in the background, sleep for 30 secs
# to allow the test to complete before verifying the result
# override default timeout to 35, slightly above the sleep duration 30 secs
cmd = 'sleep ' + str(bkg_timeout)
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
cmd,
timeout=(bkg_timeout + 5))
# try to kill lingering processes
for w in [w1, w2]:
if not w.IsNaples():
continue
cmd = 'killall ' + tc.iterators.command
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
cmd,
timeout=(bkg_timeout + 5))
# print the next_qpid
for w in [w1, w2]:
if not w.IsNaples():
continue
if tc.os == host.OS_TYPE_BSD:
cmd = 'sysctl dev.' + host.GetNaplesSysctl(
w.interface) + '.rdma.info.next_qpid'
elif tc.os == host.OS_TYPE_LINUX:
pci = host.GetNaplesPci(w.node_name, w.interface)
if pci is None:
continue
cmd = 'grep next_qpid /sys/kernel/debug/ionic/' + pci + '/lif0/rdma/info'
else:
continue
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
cmd,
timeout=(bkg_timeout + 5))
if tc.tcpdump == True:
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, "sleep 5")
tshark_cmd = "sudo tshark -r rdma_capture.pcap -T fields -e ip.addr -e infiniband.bth.opcode -e infiniband.aeth.msn"
for w in [w1, w2]:
if not w.IsNaples():
continue
api.Trigger_AddCommand(req, w.node_name, w.workload_name,
"sudo killall tcpdump")
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
tshark_cmd,
timeout=60)
#if dcqcn was enabled, disable it at the end of the test
if enable_dcqcn == True:
for w in [w1, w2]:
if not w.IsNaples():
continue
if tc.os == host.OS_TYPE_BSD:
cmd = 'sysctl sys.class.infiniband.' + host.GetNaplesSysClassSysctl(
w.interface) + '.dcqcn.match_default="0"'
elif tc.os == host.OS_TYPE_LINUX:
cmd = 'echo 0 > /sys/class/infiniband/' + host.GetNaplesSysClassSysctl(
w.interface) + '/dcqcn/match_default'
else:
continue
api.Trigger_AddCommand(req,
w.node_name,
w.workload_name,
cmd,
timeout=120)
#==============================================================
# trigger the request
#==============================================================
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
return api.types.status.SUCCESS
def Trigger(tc):
max_pings = int(getattr(tc.args, "max_pings", 60))
num_runs = int(getattr(tc.args, "num_runs", 1))
serverCmd = None
clientCmd = None
mode = tc.initial_mode
IPERF_TIMEOUT = 86400
try:
serverReq = api.Trigger_CreateExecuteCommandsRequest()
clientReq = api.Trigger_CreateExecuteCommandsRequest()
client = tc.wl_pair[0]
server = tc.wl_pair[1]
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address,
client.workload_name, client.ip_address)
num_streams = int(getattr(tc.args, "num_streams", 2))
api.Logger.info("Starting Iperf test from %s num-sessions %d"
% (tc.cmd_descr, num_streams))
if tc.iterators.proto == 'tcp':
port = api.AllocateTcpPort()
serverCmd = iperf.ServerCmd(port, jsonOut=True, run_core=3)
clientCmd = iperf.ClientCmd(server.ip_address, port, time=IPERF_TIMEOUT,
jsonOut=True, num_of_streams=num_streams,
run_core=3)
else:
port = api.AllocateUdpPort()
serverCmd = iperf.ServerCmd(port, jsonOut=True, run_core=3)
clientCmd = iperf.ClientCmd(server.ip_address, port, proto='udp',
jsonOut=True, num_of_streams=num_streams,
run_core=3)
tc.serverCmd = serverCmd
tc.clientCmd = clientCmd
api.Trigger_AddCommand(serverReq, server.node_name, server.workload_name,
serverCmd, background=True, timeout=IPERF_TIMEOUT)
api.Trigger_AddCommand(clientReq, client.node_name, client.workload_name,
clientCmd, background=True, timeout=IPERF_TIMEOUT)
tc.server_resp = api.Trigger(serverReq)
time.sleep(5)
tc.iperf_client_resp = api.Trigger(clientReq)
for _i in range(num_runs):
RF = get_redfish_obj(tc.cimc_info, mode=mode)
obs_mode = get_nic_mode(RF)
api.Logger.info("Iteration %d: curr_mode %s" % (_i, obs_mode))
if mode != obs_mode:
raise RuntimeError("Expected NIC mode %s, observed %s" % (mode, obs_mode))
next_mode = "dedicated" if mode == "ncsi" else "ncsi"
if next_mode == "ncsi":
ret = set_ncsi_mode(RF, mode="dhcp")
else:
ret = set_dedicated_mode(RF, mode="dhcp")
if ret != api.types.status.SUCCESS:
api.Logger.error("Mode switch from %s -> %s failed" %(mode, next_mode))
return api.types.status.FAILURE
api.Logger.info("Switched mode to %s" % (next_mode))
time.sleep(5)
if ret == api.types.status.SUCCESS:
curr_ilo_ip = tc.ilo_ip if next_mode == "dedicated" else tc.ilo_ncsi_ip
ret = ping(curr_ilo_ip, max_pings)
if ret != api.types.status.SUCCESS:
RF.logout()
raise RuntimeError('Unable to ping ILO, Port Switch fail from'
' %s -> %s' % (mode, next_mode))
api.Logger.info("Mode switch from %s -> %s successful" % (mode, next_mode))
else:
raise RuntimeError('Mode switch config failed')
mode = next_mode
except:
api.Logger.error(traceback.format_exc())
return api.types.status.FAILURE
return api.types.status.SUCCESS
def TestPing(tc, input_type, ipaf, pktsize, interval=0.2, count=20, deadline=0,
pktlossverif=False, background=False, hping3=False):
if input_type == 'user_input':
api.Logger.info("user_input")
elif input_type == 'local_only':
api.Logger.info("local_only test")
tc.workload_pairs = api.GetLocalWorkloadPairs()
elif input_type == 'remote_only':
api.Logger.info("remote_only test")
tc.workload_pairs = api.GetRemoteWorkloadPairs()
else:
tc.workload_pairs = api.GetLocalWorkloadPairs()
tc.workload_pairs += api.GetRemoteWorkloadPairs()
if len(tc.workload_pairs) == 0:
api.Logger.info("Skipping Testcase due to no workload pairs.")
tc.skip = True
if api.GetConfigNicMode() == 'hostpin' and ipaf == 'ipv6':
api.Logger.info("Skipping Testcase: IPv6 not supported in hostpin mode.")
return api.types.status.SUCCESS
req = None
if not api.IsSimulation():
req = api.Trigger_CreateAllParallelCommandsRequest()
else:
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
tc.cmd_cookies = []
deadline_str = ""
count_str = ""
cmd_timeout = 60
if deadline:
deadline_str = "-w {}".format(deadline)
cmd_timeout = deadline + 5
else:
count_str = "-c {}".format(count)
cmd_timeout = int(count * interval) + 5
cmd_timeout = 60 if cmd_timeout < 60 else cmd_timeout
for pair in tc.workload_pairs:
w1 = pair[0]
w2 = pair[1]
if ipaf == 'ipv6':
cmd_cookie = "%s(%s) --> %s(%s)" %\
(w1.workload_name, w1.ipv6_address, w2.workload_name, w2.ipv6_address)
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
"sudo ping6 -q -i %s %s -s %d %s %s" % (interval, count_str,
pktsize, w2.ipv6_address, deadline_str),
background=background, timeout=cmd_timeout)
else:
cmd_cookie = "%s(%s) --> %s(%s) pktsize: %s, count: %s, interval: %s, deadline: %s, background: %s" %\
(w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address, pktsize, count, interval,
deadline, background)
if hping3:
# working around hping --quiet mode issue.
cmd = f"sudo hping3 --icmp -i u{interval*1000000} {count_str} -d {pktsize} {w2.ip_address} 2> out 1> /dev/null && cat out"
else:
cmd = f"sudo ping -q -i {interval} {count_str} -s {pktsize} {w2.ip_address} {deadline_str}"
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, cmd, background=background, timeout=cmd_timeout)
api.Logger.info("Ping test from %s" % (cmd_cookie))
tc.cmd_cookies.append(cmd_cookie)
tc.resp = api.Trigger(req)
if tc.resp is None:
return api.types.status.FAILURE
result = api.types.status.SUCCESS
if not background:
cookie_idx = 0
for cmd in tc.resp.commands:
api.Logger.info("ping results for %s" % (tc.cmd_cookies[cookie_idx]))
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
if api.GetConfigNicMode() == 'hostpin' and pktsize > 1024:
result = api.types.status.SUCCESS
else:
result = api.types.status.FAILURE
elif pktlossverif:
ping_result = parsePingResult(cmd.stdout)
if not ping_result:
api.Logger.info("ping failed in packet loss verification")
result = api.types.status.FAILURE
break
cookie_idx += 1
else:
tc.bg_cmd_cookies = tc.cmd_cookies
tc.bg_cmd_resp = tc.resp
return result
def Trigger(tc):
triplet = GetThreeWorkloads()
server = triplet[0][0]
client1 = triplet[0][1]
client2 = triplet[0][2]
tc.cmd_cookies = []
naples = server
if not server.IsNaples():
naples = client1
if not client1.IsNaples():
naples = client2
if not client2.IsNaples():
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address, client1.workload_name, client1.ip_address)
api.Logger.info("Starting RTSP test from %s" % (tc.cmd_descr))
dir_path = os.path.dirname(os.path.realpath(__file__))
fullpath = dir_path + '/' + "small.vob"
api.Logger.info("fullpath %s" % (fullpath))
resp = api.CopyToWorkload(server.node_name, server.workload_name, [fullpath], 'rtspdir')
if resp is None:
return api.types.status.FAILURE
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"ls -al | grep video")
tc.cmd_cookies.append("Before RTSP")
server_cmd = "cd rtspdir && vobStreamer -p 2004 small.vob"
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
server_cmd, background = True)
tc.cmd_cookies.append("Run RTSP server")
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"openRTSP rtsp://%s:2004/vobStream" % server.ip_address)
tc.cmd_cookies.append("Run RTSP client1")
## Add Naples command validation
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show session --alg rtsp")
tc.cmd_cookies.append("show session RTSP established")
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show nwsec flow-gate | grep RTSP")
tc.cmd_cookies.append("show flow-gate")
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"ls -al | grep video")
tc.cmd_cookies.append("After RTSP")
trig_resp = api.Trigger(req)
dest = trig_resp.commands[2].stdout.find("Transport: RTP/AVP;multicast;destination=")
dest += 41
ip = trig_resp.commands[2].stdout[dest:dest+35]
end = ip.find(";")
req2 = api.Trigger_CreateExecuteCommandsRequest(serial = True)
api.Trigger_AddCommand(req2, client2.node_name, client2.workload_name,
"ip route add 224.0.0.0/4 dev %s"%(client2.interface), background = True)
tc.cmd_cookies.append("ip route client2")
api.Trigger_AddCommand(req2, client2.node_name, client2.workload_name,
"sed -i 's/geteuid/getppid/' /usr/bin/vlc && cvlc -vvv rtp://%s:8888 --start-time=00 --run-time=5" % ip[0:end])
tc.cmd_cookies.append("Run RTP client2")
api.Trigger_AddNaplesCommand(req2, naples.node_name,
"/nic/bin/halctl show session --srcip %s "%(client2.ip_address))
tc.cmd_cookies.append("show session RTP")
trig_resp2 = api.Trigger(req2)
term_resp2 = api.Trigger_TerminateAllCommands(trig_resp2)
tc.resp2 = api.Trigger_AggregateCommandsResponse(trig_resp2, term_resp2)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
ForceReleasePort("554/tcp", server)
ForceReleasePort("554/tcp", client1)
ForceReleasePort("554/tcp", client2)
return api.types.status.SUCCESS
def SetupFTPServer(node,
workload,
promiscous=False,
bind_fail=False,
listen_port=FTP_STANDARD_PORT,
restart=True):
ftpdata = dir_path + '/' + "ftp_server.txt"
api.Logger.info("fullpath %s" % (ftpdata))
resp = api.CopyToWorkload(node, workload, [ftpdata], 'ftpdir')
if resp is None:
return None
if promiscous == True:
if bind_fail == True:
vsftpd_file = "lftp_vsftpd_bind_fail.conf"
vsftpd_conf = dir_path + '/' + "lftp_vsftpd_bind_fail.conf"
else:
vsftpd_file = "lftp_vsftpd.conf"
vsftpd_conf = dir_path + '/' + "lftp_vsftpd.conf"
else:
vsftpd_file = "vsftpd.conf"
vsftpd_conf = dir_path + '/' + "vsftpd.conf"
resp = api.CopyToWorkload(node, workload, [vsftpd_conf], 'ftpdir')
if resp is None:
return None
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddCommand(
req, node, workload,
"cp ftpdir/%s /etc/vsftpd/vsftpd.conf" % (vsftpd_file))
if listen_port != FTP_STANDARD_PORT:
api.Trigger_AddCommand(
req, node, workload,
"echo \"listen_port=%d\" | tee -a /etc/vsftpd/vsftpd.conf" %
(listen_port))
api.Trigger_AddCommand(req, node, workload,
"useradd -m -c \"admin\" -s /bin/bash admin")
api.Trigger_AddCommand(req, node, workload,
"useradd -m -c \"admin1\" -s /bin/bash admin1")
api.Trigger_AddCommand(req, node, workload,
"echo \"admin\" | tee -a /etc/vsftpd.userlist")
api.Trigger_AddCommand(req, node, workload,
"echo \"admin1\" | tee -a /etc/vsftpd.userlist")
api.Trigger_AddCommand(req, node, workload,
"echo \"linuxpassword\" | passwd --stdin admin")
api.Trigger_AddCommand(req, node, workload,
"echo \"docker\" | passwd --stdin admin1")
api.Trigger_AddCommand(req, node, workload,
"echo \"docker\" | passwd --stdin admin1")
api.Trigger_AddCommand(
req, node, workload,
"mkdir /home/admin/ftp && mv ftpdir/ftp_server.txt /home/admin/ftp")
api.Trigger_AddCommand(
req, node, workload,
"touch /home/admin/ftp/ftp_client.txt && chmod 666 /home/admin/ftp/ftp_client.txt"
)
if restart == True:
api.Trigger_AddCommand(req, node, workload, "systemctl stop vsftpd")
api.Trigger_AddCommand(req, node, workload, "systemctl start vsftpd")
api.Trigger_AddCommand(req, node, workload, "systemctl enable vsftpd")
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
for cmd in trig_resp.commands:
api.PrintCommandResults(cmd)
return api.types.status.SUCCESS
def __runTraffic(intf):
api.Logger.info("Run traffic: %s" % intf)
client = None
server = None
clientCmd = None
serverCmd = None
clientReq = None
serverReq = None
for pairs in api.GetRemoteWorkloadPairs():
client = pairs[0]
api.Logger.error("Comparing client interface %s with %s" %
(client.interface, intf))
if client.interface == intf:
server = pairs[1]
break
if server is None:
api.Logger.error("No workload found for interface %s" % intf)
return api.types.status.FAILURE
cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address,\
client.workload_name, client.ip_address)
api.Logger.info("Starting Iperf test from %s" % cmd_descr)
port = api.AllocateTcpPort()
serverCmd = iperf.ServerCmd(port)
clientCmd = iperf.ClientCmd(server.ip_address, port)
serverReq = api.Trigger_CreateExecuteCommandsRequest(serial=False)
api.Trigger_AddCommand(serverReq,
server.node_name,
server.workload_name,
serverCmd,
background=True)
clientReq = api.Trigger_CreateExecuteCommandsRequest(serial=False)
api.Trigger_AddCommand(clientReq, client.node_name, client.workload_name,
clientCmd)
# Server runs in the background
server_resp = api.Trigger(serverReq)
# Sleep for some time as bg may not have been started.
time.sleep(5)
client_resp = api.Trigger(clientReq)
# Stop the backgrounded server
term_resp = api.Trigger_TerminateAllCommands(server_resp)
# We don't bother checking the iperf results, we just wanted traffic
# so just check that the commands succeeded
ret = api.types.status.SUCCESS
for cmd in server_resp.commands:
if cmd.exit_code != 0:
ret = api.types.status.FAILURE
for cmd in client_resp.commands:
if cmd.exit_code != 0:
ret = api.types.status.FAILURE
for cmd in term_resp.commands:
if cmd.exit_code != 0:
ret = api.types.status.FAILURE
return ret
def Trigger(tc):
pairs = api.GetLocalWorkloadPairs()
server = pairs[0][0]
client = pairs[0][1]
tc.cmd_cookies = []
naples = server
if not server.IsNaples():
naples = client
if not client.IsNaples():
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address, client.workload_name, client.ip_address)
api.Logger.info("Starting SUNRPC test from %s" % (tc.cmd_descr))
SetupNFSServer(server, client)
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
"sh -c 'ls -al /home/sunrpcmntdir | grep sunrpc_file.txt'")
tc.cmd_cookies.append("Before rpc")
api.Trigger_AddCommand(req, client.node_name, client.workload_name,
"sudo sh -c 'mkdir -p /home/sunrpcdir && mount %s:/home/sunrpcmntdir /home/sunrpcdir' "%(server.ip_address))
tc.cmd_cookies.append("Create mount point")
api.Trigger_AddCommand(req, client.node_name, client.workload_name,
"sudo chmod 777 /home/sunrpcdir")
tc.cmd_cookies.append("add permission")
api.Trigger_AddCommand(req, client.node_name, client.workload_name,
"mv sunrpcdir/sunrpc_file.txt /home/sunrpcdir/")
tc.cmd_cookies.append("Create file")
api.Trigger_AddCommand(req, client.node_name, client.workload_name,
"ls -al /home/sunrpcdir")
tc.cmd_cookies.append("verify file")
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
"ls -al /home/sunrpcmntdir/")
tc.cmd_cookies.append("After rpc")
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
"sh -c 'cat /home/sunrpcmntdir/sunrpc_file.txt'")
tc.cmd_cookies.append("After rpc")
# Add Naples command validation
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show session --alg sun_rpc")
tc.cmd_cookies.append("show session")
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show nwsec flow-gate")
tc.cmd_cookies.append("show security flow-gate")
# Get the timeout from the config
api.Trigger_AddNaplesCommand(req, naples.node_name, "sleep 120", timeout=120)
tc.cmd_cookies.append("sleep")
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show nwsec flow-gate | grep SUN_RPC")
tc.cmd_cookies.append("After flow-gate ageout")
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
CleanupNFSServer(server, client)
return api.types.status.SUCCESS
def Trigger(tc):
if tc.skip == True:
return api.types.status.FAILURE
#
# Set-up Test Environment
#
tc.cmd_cookies = []
cmd_cookie = "%s(%s) --> %s(%s)" %\
(tc.server.workload_name, tc.server.ip_address,
tc.client.workload_name, tc.client.ip_address)
api.Logger.info("Starting Single-IPv4-ICMP-Flow-Drops test from %s" %\
(cmd_cookie))
#
# Start TCPDUMP in background on Server/Client
#
req1 = api.Trigger_CreateExecuteCommandsRequest(serial = True)
#cmd_cookie = "start tcpdump on Server"
#cmd = "sudo tcpdump -nnSXi {} > out.txt".format(tc.server.interface)
#add_command(tc, req1, cmd_cookie, cmd, tc.server, True)
#cmd_cookie = "start tcpdump on Client"
#cmd = "sudo tcpdump -nnSXi {} > out.txt".format(tc.client.interface)
#add_command(tc, req1, cmd_cookie, cmd, tc.client, True)
#
# Start with a clean slate by clearing all sessions/flows
#
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session"
add_naples_command(tc, req1, cmd_cookie, cmd, tc.naples)
#
# Make sure that Client<=>Server Forwarding is set up
#
cmd_cookie = "trigger ping: Create case"
cmd = "ping -c1 %s -I %s" %\
(tc.server.ip_address, tc.client.interface)
add_command(tc, req1, cmd_cookie, cmd, tc.client, False)
#
# Send Good-Data from Client with ICMP-type=4,6 and Non-zero TTL
#
cmd_cookie = "send good data from Client for ICMP-Type=4: Create case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send good data from Client for ICMP-Type=6: Create case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send good data from Server for ICMP-Type=4: Create case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.server, False)
cmd_cookie = "send good data from Server for ICMP-Type=6: Create case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.server, False)
#
# Send Bad-Data (TTL=0) from Client with ICMP-type=4,6
#
cmd_cookie = "send bad data from Client for ICMP-Type=4 TTL=0: Create case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Client for ICMP-Type=6 TTL=0: Create case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server for ICMP-Type=4 TTL=0: Create case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.server, False)
cmd_cookie = "send bad data from Server for ICMP-Type=6 TTL=0: Create case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req1, cmd_cookie, cmd, tc.server, False)
#
# Do "show session" command
#
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req1, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Create case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req1, cmd_cookie, cmd, tc.naples)
tc.resp1 = api.Trigger(req1)
for command in tc.resp1.commands:
api.PrintCommandResults(command)
#
# Clearing all sessions/flows
#
req2 = api.Trigger_CreateExecuteCommandsRequest(serial = True)
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session"
add_naples_command(tc, req2, cmd_cookie, cmd, tc.naples)
#
# Make sure that Client<=>Server Forwarding is set up
#
cmd_cookie = "trigger ping: Re-use case"
cmd = "ping -c1 %s -I %s" %\
(tc.server.ip_address, tc.client.interface)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
#
# Re-send Good-Data from Client with ICMP-type=4,6 and Non-zero TTL
#
cmd_cookie = "re-send good data from Client for ICMP-Type=4: Re-use case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "re-send good data from Client for ICMP-Type=6: Re-use case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "re-send good data from Server for ICMP-Type=4: Re-use case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
cmd_cookie = "re-send good data from Server for ICMP-Type=6: Re-use case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
#
# Re-send Bad-Data (TTL=0) from Client with ICMP-type=4,6
#
cmd_cookie = "re-send bad data from Client for ICMP-Type=4 TTL=0: Re-use case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "re-send bad data from Client for ICMP-Type=6 TTL=0: Re-use case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "re-send bad data from Server for ICMP-Type=4 TTL=0: Re-use case"
cmd = "hping3 --icmp --icmptype 4 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
cmd_cookie = "re-send bad data from Server for ICMP-Type=6 TTL=0: Re-use case"
cmd = "hping3 --icmp --icmptype 6 --force-icmp --ttl 0 --count 1 {}"\
.format(tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
#
# Do "show session" command
#
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req2, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Re-use case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req2, cmd_cookie, cmd, tc.naples)
tc.resp2 = api.Trigger(req2)
for command in tc.resp2.commands:
api.PrintCommandResults(command)
#
# Clearing all sessions/flows and Sleep for 45secs
#
req3 = api.Trigger_CreateExecuteCommandsRequest(serial = True)
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session; sleep 45"
add_naples_command(tc, req3, cmd_cookie, cmd, tc.naples)
#
# Do "show session" command
#
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req3, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Delete case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req3, cmd_cookie, cmd, tc.naples)
tc.resp3 = api.Trigger(req3)
for command in tc.resp3.commands:
api.PrintCommandResults(command)
return api.types.status.SUCCESS
def Setup(tc):
result = api.types.status.SUCCESS
tc.workload_pairs = []
tc.skip = False
tc.sleep = getattr(tc.args, "sleep", 200)
tc.allowed_down_time = getattr(tc.args, "allowed_down_time", 0)
tc.pkg_name = getattr(tc.args, "naples_upgr_pkg", "naples_fw.tar")
tc.node_selection = tc.iterators.selection
tc.iperf = getattr(tc.args, "iperf", False)
tc.trex = getattr(tc.args, "trex", False)
tc.background_ping = getattr(tc.args, "background_ping", True)
tc.failure_stage = getattr(tc.args, "failure_stage", None)
tc.failure_reason = getattr(tc.args, "failure_reason", None)
if tc.node_selection not in ["any", "all"]:
api.Logger.error(
"Incorrect Node selection option {} specified. Use 'any' or 'all'".
format(tc.node_selection))
tc.skip = True
return api.types.status.FAILURE
tc.nodes = api.GetNaplesHostnames()
if tc.node_selection == "any":
tc.nodes = [random.choice(tc.nodes)]
if len(tc.nodes) == 0:
api.Logger.error("No naples nodes found")
result = api.types.status.FAILURE
api.Logger.info("Running ISSU upgrade test on %s" % (", ".join(tc.nodes)))
req = api.Trigger_CreateExecuteCommandsRequest()
for node in tc.nodes:
api.Trigger_AddNaplesCommand(
req, node, "rm -rf /data/techsupport/DSC_TechSupport_*")
api.Trigger_AddNaplesCommand(req, node,
"rm -rf /update/pds_upg_status.txt")
api.Trigger_AddNaplesCommand(
req, node, "touch /data/upgrade_to_same_firmware_allowed && sync")
api.Trigger_AddNaplesCommand(req, node,
"touch /data/no_watchdog && sync")
api.Trigger_AddNaplesCommand(req, node, "/nic/tools/fwupdate -r")
resp = api.Trigger(req)
for cmd_resp in resp.commands:
api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
api.Logger.error("Setup failed %s", cmd_resp.command)
tc.skip = True
return api.types.status.FAILURE
if upgrade_utils.ResetUpgLog(tc.nodes) != api.types.status.SUCCESS:
api.Logger.error("Failed in Reseting Upgrade Log files.")
return api.types.status.FAILURE
# verify mgmt connectivity
result = traffic.VerifyMgmtConnectivity(tc.nodes)
if result != api.types.status.SUCCESS:
api.Logger.error("Failed in Mgmt Connectivity Check during Setup.")
tc.skip = True
return result
if tc.failure_stage != None:
result = upgrade_utils.HitlessNegativeSetup(tc)
if result != api.types.status.SUCCESS:
return result
# choose workloads for connectivity/traffic test
result = ChooseWorkLoads(tc)
if result != api.types.status.SUCCESS or tc.skip:
api.Logger.error("Failed to Choose Workloads.")
return result
# verify endpoint connectivity
if VerifyConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check during Setup.")
if not SKIP_CONNECTIVITY_FAILURE:
result = api.types.status.FAILURE
tc.skip = True
return result
# setup packet test based on upgrade_mode
result = PacketTestSetup(tc)
if result != api.types.status.SUCCESS or tc.skip:
api.Logger.error("Failed in Packet Test setup.")
return result
# Update security profile
if utils.UpdateSecurityProfileTimeouts(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed to update the security profile")
tc.skip = True
result = api.types.status.FAILURE
api.Logger.info(f"Upgrade: Setup returned {result}")
return result
def ConfigWorkloadSecondaryIp(workload, is_add, sec_ip_count_per_intf=1):
res = api.types.status.SUCCESS
wl_sec_ip_list = []
if (workload.uplink_vlan != 0):
return wl_sec_ip_list
nodes = api.GetWorkloadNodeHostnames()
max_untag_wl = 0
max_tag_wl = 0
if is_add == True:
op = "add"
else:
op = "del"
is_wl_bm_type = False
for node in nodes:
if api.IsBareMetalWorkloadType(node):
is_wl_bm_type = True
workloads = api.GetWorkloads(node)
num_untag_wl_in_node = 0
num_tag_wl_in_node = 0
for wl in workloads:
if (wl.uplink_vlan == 0):
num_untag_wl_in_node += 1
else:
num_tag_wl_in_node += 1
if num_untag_wl_in_node > max_untag_wl:
max_untag_wl = num_untag_wl_in_node
if num_tag_wl_in_node > max_tag_wl:
max_tag_wl = num_tag_wl_in_node
#api.Logger.info("Node {} WL #untag {} #tag {} ".format(node, num_untag_wl_in_node, num_tag_wl_in_node))
#api.Logger.info("Topo Max untag WL {} Max tag WL {} ".format(max_untag_wl, max_tag_wl))
sec_ip_incr_step = max_untag_wl
if is_wl_bm_type == False:
sec_ip_incr_step += max_tag_wl
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
wl = workload
sec_ipv4_allocator = resmgr.IpAddressStep(
wl.ip_prefix.split('/')[0],
str(ipaddress.IPv4Address(sec_ip_incr_step)), sec_ip_count_per_intf)
sec_ip4_addr_str = str(sec_ipv4_allocator.Alloc())
for i in range(sec_ip_count_per_intf):
sec_ip4_addr_str = str(sec_ipv4_allocator.Alloc())
sec_ip_prefix = sec_ip4_addr_str + "/" + str(
wl.ip_prefix.split('/')[1])
wl_sec_ip_list.append(sec_ip4_addr_str)
#api.Logger.info("Node {} WL {} Intf {} Secondary IP {} Op {}".format(
# wl.node_name, wl.workload_name, wl.interface, sec_ip_prefix, op))
api.Trigger_AddCommand(
req, wl.node_name, wl.workload_name,
"ip address %s %s dev %s " % (op, sec_ip_prefix, wl.interface))
trig_resp = api.Trigger(req)
#api.Logger.info("Response ")
#for cmd in trig_resp.commands:
# api.PrintCommandResults(cmd)
return wl_sec_ip_list
def Trigger(tc):
if tc.ignore == True:
return api.types.status.SUCCESS
if tc.error == True:
return api.types.status.FAILURE
protoDir1 = api.GetTopologyDirectory() +\
"/gen/telemetry/{}/{}".format('mirror', tc.iterators.proto)
api.Logger.info("Template Config files location: ", protoDir1)
protoDir2 = api.GetTopologyDirectory() +\
"/gen/telemetry/{}/{}".format('flowmon', tc.iterators.proto)
api.Logger.info("Template Config files location: ", protoDir2)
result = api.types.status.SUCCESS
count = 0
MirrorPolicies = utils.GetTargetJsons('mirror', tc.iterators.proto)
FlowMonPolicies = utils.GetTargetJsons('flowmon', tc.iterators.proto)
#mirror_policy_idx = 0
flowmon_policy_idx = 0
ret_count = 0
for mirror_json in MirrorPolicies:
#
# Get template-Mirror Config
#
newMirrorObjects = agent_api.AddOneConfig(mirror_json)
if len (newMirrorObjects) == 0:
api.Logger.error("Adding new Mirror objects to store failed")
tc.error = True
return api.types.status.FAILURE
agent_api.RemoveConfigObjects(newMirrorObjects)
#
# Ignore Multi-collector template config's, since Expanded-Telemetry
# testbundle dynamically creates such config's
#
if len(newMirrorObjects[0].spec.collectors) > 1:
continue
idx = 0
for flowmon_json in FlowMonPolicies:
if idx < flowmon_policy_idx:
idx += 1
continue
#
# Get template-FlowMon Config
#
newFlowMonObjects = agent_api.AddOneConfig(flowmon_json)
if len (newFlowMonObjects) == 0:
api.Logger.error("Adding new FlowMon objects to store failed")
tc.error = True
return api.types.status.FAILURE
agent_api.RemoveConfigObjects(newFlowMonObjects)
#
# Ignore Multi-collector template config's, since Expanded-Telemetry
# testbundle dynamically creates such config's
#
if len(newFlowMonObjects[0].spec.exports) > 1:
flowmon_policy_idx += 1
idx += 1
continue
#
# Modify template-Mirror / template-FlowMon Config to make sure that
# Naples-node # act as either source or destination
#
# Set up Collector in the remote node
#
eutils.generateMirrorConfig(tc, mirror_json, newMirrorObjects)
eutils.generateFlowMonConfig(tc, flowmon_json, newFlowMonObjects)
ret_count = 0
for i in range(0, len(tc.mirror_verif)):
#
# If Execution-Optimization is enabled, no need to run the test
# for the same protocol more than once
#
if i > 0 and tc.mirror_verif[i]['protocol'] ==\
tc.mirror_verif[i-1]['protocol']:
continue
#
# Flow-ERSPAN/FlowMon for TCP-traffic is not tested (yet) in
# Classic-mode until applicable pkt-trigger tools are identified
#
if tc.classic_mode == True and\
tc.mirror_verif[i]['protocol'] == 'tcp':
continue
#
# Push Mirror / FlowMon Config to Naples
#
ret = agent_api.PushConfigObjects(newMirrorObjects,
[tc.naples.node_name], [tc.naples_device_name])
if ret != api.types.status.SUCCESS:
api.Logger.error("Unable to push mirror objects")
tc.error = True
return api.types.status.FAILURE
ret = agent_api.PushConfigObjects(newFlowMonObjects,
[tc.naples.node_name], [tc.naples_device_name])
if ret != api.types.status.SUCCESS:
agent_api.DeleteConfigObjects(newMirrorObjects,
[tc.naples.node_name], [tc.naples_device_name])
api.Logger.error("Unable to push flowmon objects")
tc.error = True
return api.types.status.FAILURE
#
# Establish Forwarding set up between Naples-peer and Collectors
#
eutils.establishForwardingSetup(tc)
#
# Give a little time for flows clean-up to happen so that
# stale IPFIX records don't show up
#
if tc.classic_mode == True:
time.sleep(1)
req_tcpdump_erspan = api.Trigger_CreateExecuteCommandsRequest(\
serial = True)
req_tcpdump_flowmon = api.Trigger_CreateExecuteCommandsRequest(\
serial = True)
for c in range(0, len(tc.flow_collector)):
#
# Set up TCPDUMP's on the collector
#
idx = tc.flow_collector_idx[c]
if tc.flow_collector[c].IsNaples():
cmd = "tcpdump -c 600 -XX -vv -nni {} ip proto gre\
and dst {} --immediate-mode -U -w flow-mirror-{}.pcap"\
.format(tc.flow_collector[c].interface,
tc.collector_ip_address[idx], c)
else:
cmd = "tcpdump -p -c 600 -XX -vv -nni {} ip proto gre\
and dst {} --immediate-mode -U -w flow-mirror-{}.pcap"\
.format(tc.flow_collector[c].interface,
tc.collector_ip_address[idx], c)
eutils.add_command(req_tcpdump_erspan,
tc.flow_collector[c], cmd, True)
for c in range(0, len(tc.flowmon_collector)):
idx = tc.flowmon_collector_idx[c]
if tc.flowmon_collector[c].IsNaples():
cmd = "tcpdump -c 600 -XX -vv -nni {} udp and\
dst port {} and dst {} --immediate-mode\
-U -w flowmon-{}.pcap"\
.format(tc.flowmon_collector[c].interface,
tc.export_port[c], tc.collector_ip_address[idx], c)
else:
cmd = "tcpdump -p -c 600 -XX -vv -nni {} udp and\
dst port {} and dst {} --immediate-mode\
-U -w flowmon-{}.pcap"\
.format(tc.flowmon_collector[c].interface,
tc.export_port[c], tc.collector_ip_address[idx], c)
eutils.add_command(req_tcpdump_flowmon,
tc.flowmon_collector[c], cmd, True)
resp_tcpdump_erspan = api.Trigger(req_tcpdump_erspan)
for cmd in resp_tcpdump_erspan.commands:
api.PrintCommandResults(cmd)
#
# Classic mode requires a delay to make sure that TCPDUMP
# background process is fully up
#
if tc.classic_mode == True:
time.sleep(2)
resp_tcpdump_flowmon = api.Trigger(req_tcpdump_flowmon)
for cmd in resp_tcpdump_flowmon.commands:
api.PrintCommandResults(cmd)
#
# Classic mode requires a delay to make sure that TCPDUMP
# background process is fully up
#
if tc.classic_mode == True:
time.sleep(2)
#
# Trigger packets for ERSPAN / FLOWMON to take effect
#
tc.protocol = tc.mirror_verif[i]['protocol']
tc.dest_port = utils.GetDestPort(tc.mirror_verif[i]['port'])
if api.GetNodeOs(tc.naples.node_name) == 'linux':
eutils.triggerTrafficInClassicModeLinux(tc)
else:
eutils.triggerTrafficInHostPinModeOrFreeBSD(tc)
#
# Dump sessions/flows/P4-tables for debug purposes
#
eutils.showSessionAndP4TablesForDebug(tc, tc.flow_collector,
tc.flow_collector_idx)
#
# Terminate TCPDUMP background process
#
term_resp_tcpdump_erspan = api.Trigger_TerminateAllCommands(\
resp_tcpdump_erspan)
tc.resp_tcpdump_erspan = api.Trigger_AggregateCommandsResponse(\
resp_tcpdump_erspan, term_resp_tcpdump_erspan)
term_resp_tcpdump_flowmon = api.Trigger_TerminateAllCommands(\
resp_tcpdump_flowmon)
tc.resp_tcpdump_flowmon =api.Trigger_AggregateCommandsResponse(\
resp_tcpdump_flowmon, term_resp_tcpdump_flowmon)
# Delete the objects
agent_api.DeleteConfigObjects(newMirrorObjects,
[tc.naples.node_name], [tc.naples_device_name])
agent_api.DeleteConfigObjects(newFlowMonObjects,
[tc.naples.node_name], [tc.naples_device_name])
#
# Make sure that Mirror/Flowmon-config has been removed
#
tc.resp_cleanup = eutils.showP4TablesForValidation(tc)
#
# Validate ERSPAN packets reception
#
tc.tcp_erspan_pkts_expected = \
NUMBER_OF_TCP_ERSPAN_PACKETS_PER_SESSION
tc.udp_erspan_pkts_expected = (tc.udp_count << 1)
tc.icmp_erspan_pkts_expected = (tc.icmp_count << 1)
protocol = tc.protocol
if tc.protocol == 'udp' and tc.iterators.proto == 'mixed':
tc.protocol = 'udp-mixed'
tc.icmp_erspan_pkts_expected = tc.udp_erspan_pkts_expected
tc.feature = 'flow-erspan'
res_1 = eutils.validateErspanPackets(tc, tc.flow_collector,
tc.flow_collector_idx)
#
# Validate IPFIX packets reception
#
tc.protocol = protocol
tc.feature = 'flowmon'
res_2 = eutils.validateIpFixPackets(tc)
#
# Validate Config-cleanup
#
res_3 = eutils.validateConfigCleanup(tc)
if res_1 == api.types.status.FAILURE or\
res_2 == api.types.status.FAILURE or\
res_3 == api.types.status.FAILURE:
result = api.types.status.FAILURE
if result == api.types.status.FAILURE:
break
ret_count += 1
flowmon_policy_idx += 1
break
if result == api.types.status.FAILURE:
break
count += ret_count
tc.SetTestCount(count)
return result
def Trigger(tc):
if tc.skip: return api.types.status.SUCCESS
result = api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if tc.args.mode == "non-promiscuous":
# Run tcdpump in non-promiscuous mode
tcpdump_flags_extra = " -p "
else:
tcpdump_flags_extra = ""
# Run tcpdump on all interfaces
for intf in tc.all_intfs:
if api.GetNodeOs(
tc.naples_node) == "windows" and intf in tc.host_intfs:
intfGuid = ionic_utils.winIntfGuid(tc.naples_node, intf)
intfVal = str(ionic_utils.winTcpDumpIdx(tc.naples_node, intfGuid))
cmd = "/mnt/c/Windows/System32/tcpdump.exe"
else:
intfVal = intf
cmd = "tcpdump"
cmd += " -l -i " + intfVal + tcpdump_flags_extra + " -tne ether host " + tc.random_mac
__PR_AddCommand(intf, tc, req, cmd, True)
cmd = "sleep 1; ping -c 5 " + tc.target_IP + ";sleep 1"
api.Trigger_AddHostCommand(req, tc.peer_node, cmd)
trig_resp = api.Trigger(req)
# Verify packet filter flags of each interface in halctl
show_lif_resp, ret = hal_show_utils.GetHALShowOutput(tc.naples_node, "lif")
if not ret:
api.Logger.error("Something went wrong with GetHALShowOutput")
result = api.types.status.FAILURE
lif_obj_docs = yaml.load_all(show_lif_resp.commands[0].stdout)
for lif_obj in lif_obj_docs:
if lif_obj == None:
break
# See if the lif belongs to any of the interface in tc.all_intfs (inteface lif)
intf_lif = False
for intf in tc.all_intfs:
if api.GetNodeOs(
tc.naples_node) == "windows" and intf in tc.host_intfs:
halIntfName = ionic_utils.winHalIntfName(tc.naples_node, intf)
else:
halIntfName = intf
if lif_obj['spec']['name'].startswith(halIntfName):
intf_lif = True
break
lif_pr_flag = lif_obj['spec']['packetfilter']['receivepromiscuous']
# A lif must have its PR flag when it is an interface lif and tc.args.mode is 'promiscuous'
if tc.args.mode == "promiscuous":
if intf_lif and lif_pr_flag != True:
api.Logger.error(
"halctl PR flag not set for promiscuous mode interface [%s]"
% (lif_obj['spec']['name']))
result = api.types.status.FAILURE
else:
if lif_pr_flag == True:
api.Logger.error(
"halctl PR flag set for non-promiscuous mode LIF [%s]" %
(lif_obj['spec']['name']))
result = api.types.status.FAILURE
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
# Search tcpdump stdout for packets with dst MAC matching tc.random_mac
pattern = "> " + tc.random_mac
cmds = resp.commands[:-1]
for intf, cmd in zip(tc.all_intfs, cmds):
found = cmd.stdout.find(pattern)
if found > 0 and not tc.expect_pkt[intf]:
api.Logger.error(
"Interface [%s] received Unknown unicast packet while not expecting"
% (intf))
result = api.types.status.FAILURE
elif found == -1 and tc.expect_pkt[intf]:
api.Logger.error(
"Interface [%s] did not receive expected Unknown unicast packet"
% (intf))
result = api.types.status.FAILURE
# Incase of testcase failure, dump the entire command output for further debug
if result == api.types.status.FAILURE:
api.Logger.error(
" ============================= COMMAND DUMP ================================================"
)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
api.Logger.error(
" ============================= END ========================================================"
)
return result
def Trigger(tc):
if tc.args.type == 'local_only':
pairs = api.GetLocalWorkloadPairs()
else:
pairs = api.GetRemoteWorkloadPairs()
tc.cmd_cookies = []
server, client = pairs[0]
naples = server
if not server.IsNaples():
naples = client
if not client.IsNaples():
return api.types.status.SUCCESS
else:
client, server = pairs[0]
tc.client = client
tc.server = server
addPktFltrRuleOnEp(tc, True)
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "%s(%s) --> %s(%s)" %\
(server.workload_name, server.ip_address, client.workload_name, client.ip_address)
api.Logger.info("Starting UDP Aging test from %s" % (cmd_cookie))
cmd_cookie = "halctl clear session"
api.Trigger_AddNaplesCommand(req, server.node_name,
"/nic/bin/halctl clear session")
tc.cmd_cookies.append(cmd_cookie)
server_port = api.AllocateUdpPort()
timeout_str = 'udp-timeout'
if tc.args.skip_security_prof == False:
timeout = get_timeout(timeout_str)
else:
timeout = DEFAULT_UDP_TIMEOUT
#Step 0: Update the timeout in the config object
if tc.args.skip_security_prof == False:
update_timeout(timeout_str, tc.iterators.timeout)
if tc.args.skip_security_prof == False:
timeout = get_timeout(timeout_str)
else:
timeout = DEFAULT_UDP_TIMEOUT
#profilereq = api.Trigger_CreateExecuteCommandsRequest(serial = True)
#api.Trigger_AddNaplesCommand(profilereq, naples.node_name, "/nic/bin/halctl show nwsec profile --id 11")
#profcommandresp = api.Trigger(profilereq)
#cmd = profcommandresp.commands[-1]
#for command in profcommandresp.commands:
# api.PrintCommandResults(command)
#timeout = get_haltimeout(timeout_str, cmd)
#tc.config_update_fail = 0
#if (timeout != timetoseconds(tc.iterators.timeout)):
# tc.config_update_fail = 1
cmd_cookie = "start server"
api.Trigger_AddCommand(req,
server.node_name,
server.workload_name,
"sudo hping3 -9 %s" % (server_port),
background=True)
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "start client"
api.Trigger_AddCommand(
req, client.node_name, client.workload_name,
"sudo hping3 -2 %s -p %s -c 1" % (server.ip_address, server_port))
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "Before aging show session"
api.Trigger_AddNaplesCommand(
req, naples.node_name,
"/nic/bin/halctl show session --dstip %s | grep UDP" %
(server.ip_address))
tc.cmd_cookies.append(cmd_cookie)
#Get it from the config
cmd_cookie = "sleep"
api.Trigger_AddNaplesCommand(req,
naples.node_name,
"sleep %s" % (timeout + GRACE_TIME),
timeout=300)
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "After aging show session"
api.Trigger_AddNaplesCommand(
req, naples.node_name,
"/nic/bin/halctl show session --dstip %s | grep UDP" %
(server.ip_address))
tc.cmd_cookies.append(cmd_cookie)
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
return api.types.status.SUCCESS
def DeleteTunnels(self, serial=True):
req = api.Trigger_CreateExecuteCommandsRequest(serial)
for tunnel in self.__tunnels:
tunnel.Delete(req)
return self.__process_req(req)
def Trigger(tc):
serverCmd = None
clientCmd = None
IPERF_TIMEOUT = 86400
try:
serverReq = api.Trigger_CreateExecuteCommandsRequest()
clientReq = api.Trigger_CreateExecuteCommandsRequest()
client = tc.wl_pair[0]
server = tc.wl_pair[1]
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address,
client.workload_name, client.ip_address)
num_streams = int(getattr(tc.args, "num_streams", 2))
api.Logger.info("Starting Iperf test from %s num-sessions %d" %
(tc.cmd_descr, num_streams))
if tc.iterators.proto == 'tcp':
port = api.AllocateTcpPort()
serverCmd = iperf.ServerCmd(port, jsonOut=True, run_core=3)
clientCmd = iperf.ClientCmd(server.ip_address,
port,
time=IPERF_TIMEOUT,
jsonOut=True,
num_of_streams=num_streams,
run_core=3)
else:
port = api.AllocateUdpPort()
serverCmd = iperf.ServerCmd(port, jsonOut=True, run_core=3)
clientCmd = iperf.ClientCmd(server.ip_address,
port,
proto='udp',
jsonOut=True,
num_of_streams=num_streams,
run_core=3)
tc.serverCmd = serverCmd
tc.clientCmd = clientCmd
api.Trigger_AddCommand(serverReq,
server.node_name,
server.workload_name,
serverCmd,
background=True,
timeout=IPERF_TIMEOUT)
api.Trigger_AddCommand(clientReq,
client.node_name,
client.workload_name,
clientCmd,
background=True,
timeout=IPERF_TIMEOUT)
tc.server_resp = api.Trigger(serverReq)
time.sleep(5)
tc.iperf_client_resp = api.Trigger(clientReq)
_run_vmedia_traffic(tc.node_name)
except:
api.Logger.error(traceback.format_exc())
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Trigger(tc):
tc.cmd_cookies = []
pairs = api.GetRemoteWorkloadPairs()
w1 = pairs[0][0]
w2 = pairs[0][1]
if w1.IsNaples() and w2.IsNaples():
api.Logger.info("naples-naples unsupported currently for tcp-proxy")
return api.types.status.DISABLED
store_proxy_objects = netagent_cfg_api.QueryConfigs(kind='TCPProxyPolicy')
if len(store_proxy_objects) == 0:
api.Logger.error("No tcp proxy objects in store")
return api.types.status.FAILURE
ret = netagent_cfg_api.PushConfigObjects(store_proxy_objects,
ignore_error=True)
if ret != api.types.status.SUCCESS:
api.Logger.error("Unable to push tcp_proxy policies")
return api.types.status.FAILURE
get_config_objects = netagent_cfg_api.GetConfigObjects(store_proxy_objects)
if len(get_config_objects) == 0:
api.Logger.error("Unable to fetch newly pushed objects")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if w2.IsNaples():
cmd_cookie = "tcp proxy sessions on %s BEFORE running iperf traffic" % (
w2.node_name)
api.Trigger_AddNaplesCommand(req, w2.node_name,
"/nic/bin/halctl show tcp-proxy session")
tc.cmd_cookies.append(cmd_cookie)
if w1.IsNaples():
cmd_cookie = "tcp proxy sessions on %s BEFORE running iperf traffic" % (
w1.node_name)
api.Trigger_AddNaplesCommand(req, w1.node_name,
"/nic/bin/halctl show tcp-proxy session")
tc.cmd_cookies.append(cmd_cookie)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s) on tcp proxy port %s pktsize %s" %\
(w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address, tc.iterators.port, tc.iterators.pktsize)
api.Logger.info("Starting Iperf test from %s" % (tc.cmd_descr))
cmd_cookie = "Running iperf server on %s" % (w1.workload_name)
api.Trigger_AddCommand(req,
w1.node_name,
w1.workload_name,
"iperf3 -s -p %s" % (tc.iterators.port),
background=True)
tc.cmd_cookies.append(cmd_cookie)
cmd_cookie = "Running iperf client on %s" % (w2.workload_name)
api.Trigger_AddCommand(
req, w2.node_name, w2.workload_name, "iperf3 -c %s -p %s -M %s" %
(w1.ip_address, tc.iterators.port, tc.iterators.pktsize))
tc.cmd_cookies.append(cmd_cookie)
if w1.IsNaples():
cmd_cookie = "tcp proxy sessions on %s AFTER running iperf traffic" % (
w1.node_name)
api.Trigger_AddNaplesCommand(req, w1.node_name,
"/nic/bin/halctl show tcp-proxy session")
tc.cmd_cookies.append(cmd_cookie)
if w2.IsNaples():
cmd_cookie = "tcp proxy sessions on %s AFTER running iperf traffic" % (
w2.node_name)
api.Trigger_AddNaplesCommand(req, w2.node_name,
"/nic/bin/halctl show tcp-proxy session")
tc.cmd_cookies.append(cmd_cookie)
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
return api.types.status.SUCCESS
def __execute_athena_client(node, nic, tc, cmd_options):
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd = __CMDPATH + __CMDBASE + __CMDSEP + cmd_options
api.Trigger_AddNaplesCommand(req, node, cmd, nic, timeout=__CMD_TIMEOUT)
tc.resp = api.Trigger(req)
def Trigger(tc):
pkgname = os.path.basename(tc.args.package)
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.nodes:
api.Trigger_AddHostCommand(req, n, "mkdir -p %s" % api.GetHostToolsDir())
api.Trigger_AddHostCommand(req, n, "tar xf %s" % pkgname)
if tc.os == 'linux':
api.Trigger_AddHostCommand(req, n, "make modules", rundir = pnsodefs.PNSO_DRIVER_DIR, timeout=120)
else:
api.Trigger_AddHostCommand(req, n, "./freebsd_build.sh", rundir = pnsodefs.PNSO_DRIVER_DIR, timeout=120)
api.Trigger_AddHostCommand(req, n, "ls sonic.ko", rundir = pnsodefs.PNSO_DRIVER_DIR)
api.Trigger_AddHostCommand(req, n, "ls pencake.ko", rundir = pnsodefs.PNSO_DRIVER_DIR)
api.Trigger_AddHostCommand(req, n, "dmesg -c 2>&1 > /dev/null")
api.Trigger_AddHostCommand(req, n, "cp sonic.ko %s" % api.GetHostToolsDir(), rundir = pnsodefs.PNSO_DRIVER_DIR)
api.Trigger_AddHostCommand(req, n, "cp pencake.ko %s" % api.GetHostToolsDir(), rundir = pnsodefs.PNSO_DRIVER_DIR)
api.Trigger_AddHostCommand(req, n, "lspci -d 1dd8:")
if tc.os == 'linux':
api.Trigger_AddHostCommand(req, n, "rmmod pencake || true", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "rmmod sonic || true", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n,
"insmod sonic.ko core_count=%d" % tc.args.maxcpus,
rundir = pnsodefs.PNSO_DRIVER_DIR, timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "insmod pencake.ko repeat=1",
rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "sleep 40", timeout=300)
else:
api.Trigger_AddHostCommand(req, n, "kldunload pencake || true", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "kldunload sonic || true", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n,
"kenv compat.linuxkpi.sonic_core_count=%d" % tc.args.maxcpus,
rundir = pnsodefs.PNSO_DRIVER_DIR)
api.Trigger_AddHostCommand(req, n, "kldload ./sonic.ko", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "kldload ./pencake.ko", rundir = pnsodefs.PNSO_DRIVER_DIR,
timeout = int(tc.args.maxcpus) * 100)
api.Trigger_AddHostCommand(req, n, "sleep 20", timeout=300)
cmd = api.Trigger_AddHostCommand(req, n, "dmesg | tail -n 100")
tc.dmesg_commands.append(cmd)
if tc.os == 'linux':
for c in range(1, 5):
output = api.Trigger_AddHostCommand(req, n, "cat /sys/module/pencake/status/%d" % c)
tc.output_commands.append(output)
tc.succ_cmd = api.Trigger_AddHostCommand(req, n, "cat /sys/module/pencake/status/success")
tc.fail_cmd = api.Trigger_AddHostCommand(req, n, "cat /sys/module/pencake/status/fail")
else:
output = api.Trigger_AddHostCommand(req, n, "cat /dev/pencake")
tc.output_commands.append(output)
tc.succ_cmd = api.Trigger_AddHostCommand(req, n, "sysctl -n compat.linuxkpi.pencake_success_cnt")
tc.fail_cmd = api.Trigger_AddHostCommand(req, n, "sysctl -n compat.linuxkpi.pencake_fail_cnt")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
if tc.skip == True:
return api.types.status.FAILURE
#
# Set-up Test Environment
#
tc.cmd_cookies = []
tc.cookie_idx = 0
cmd_cookie = "%s(%s) --> %s(%s)" %\
(tc.server.workload_name, tc.server.ip_address,
tc.client.workload_name, tc.client.ip_address)
api.Logger.info("Starting Single-IPv4-TCP-Flow-Drops test from %s" %\
(cmd_cookie))
#
# Start TCPDUMP in background on Server/Client
#
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
#cmd_cookie = "start tcpdump on Server"
#cmd = "sudo tcpdump -nnSXi {} > out.txt".format(tc.server.interface)
#add_command(tc, req, cmd_cookie, cmd, tc.server, True)
#cmd_cookie = "start tcpdump on Client"
#cmd = "sudo tcpdump -nnSXi {} > out.txt".format(tc.client.interface)
#add_command(tc, req, cmd_cookie, cmd, tc.client, True)
#
# Start with a clean slate by clearing all sessions/flows
# 45 secs sleep is to ensure that residual FlowDrops induced by previous
# non-flowstats testbundle (in this case Connectivity) is flushed out
#
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session; sleep 45"
add_naples_command(tc, req, cmd_cookie, cmd, tc.naples)
#
# Make sure that Client<=>Server Forwarding is set up
#
cmd_cookie = "trigger ping: Create case"
cmd = "ping -c1 %s -I %s" %\
(tc.server.ip_address, tc.client.interface)
add_command(tc, req, cmd_cookie, cmd, tc.client, False)
tc.resp = api.Trigger(req)
for command in tc.resp.commands:
api.PrintCommandResults(command)
#
# Allocate TCP-portnum for Server and start the service on the Server
#
req_nc1 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.server_port = api.AllocateTcpPort()
cmd_cookie = "start server"
cmd = "nc --listen %s" % (tc.server_port)
add_command(tc, req_nc1, cmd_cookie, cmd, tc.server, True)
#
# Allocate TCP-portnum for Client and establish TCP-connection
#
tc.client_port = api.AllocateTcpPort()
cmd_cookie = "start client"
cmd = "nc {} {} --source-port {} "\
.format(tc.server.ip_address, tc.server_port, tc.client_port)
add_command(tc, req_nc1, cmd_cookie, cmd, tc.client, True)
resp_nc1 = api.Trigger(req_nc1)
for command in resp_nc1.commands:
api.PrintCommandResults(command)
#
# Do applicable "show session" commands and
# retrieve Seq-num and Ack-num associated with the session
#
req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req1, cmd_cookie, cmd, tc.naples)
cmd_cookie = "show session detail"
cmd = "/nic/bin/halctl show session --dstport {} --dstip {} --yaml"\
.format(tc.server_port, tc.server.ip_address)
add_naples_command(tc, req1, cmd_cookie, cmd, tc.naples)
tc.resp1 = api.Trigger(req1)
cmd = tc.resp1.commands[-1]
for command in tc.resp1.commands:
api.PrintCommandResults(command)
tc.pre_ctrckinf = get_conntrackinfo(cmd)
#
# Send Bad Data with TTL=0 from both Client and Server
#
req2 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "send bad data from Client TTL=0: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ttl 0 --data 10 {}".format(tc.client_port, tc.server_port,
tc.pre_ctrckinf.r_tcpacknum,
tc.pre_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TTL=0: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ttl 0 --data 10 {}".format(tc.server_port, tc.client_port,
tc.pre_ctrckinf.i_tcpacknum,
tc.pre_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
#
# Send Bad Data with TCP-RSVD-FLAGS-BIT-0 set from both Client and Server
#
cmd_cookie = "send bad data from Client TCP-RSVD-FLAGS-BIT-0: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--xmas --data 10 {}".format(tc.client_port, tc.server_port,
tc.pre_ctrckinf.r_tcpacknum + 10,
tc.pre_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TCP-RSVD-FLAGS-BIT-0: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--xmas --data 10 {}".format(tc.server_port, tc.client_port,
tc.pre_ctrckinf.i_tcpacknum + 10,
tc.pre_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
#
# Send Bad Data with TCP-RSVD-FLAGS-BIT-1 set from both Client and Server
#
cmd_cookie = "send bad data from Client TCP-RSVD-FLAGS-BIT-1: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ymas --data 10 {}".format(tc.client_port, tc.server_port,
tc.pre_ctrckinf.r_tcpacknum + 20,
tc.pre_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TCP-RSVD-FLAGS-BIT-1: Create case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ymas --data 10 {}".format(tc.server_port, tc.client_port,
tc.pre_ctrckinf.i_tcpacknum + 20,
tc.pre_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req2, cmd_cookie, cmd, tc.server, False)
tc.resp2 = api.Trigger(req2)
for command in tc.resp2.commands:
api.PrintCommandResults(command)
#
# Do "show session" command
#
req3 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cookie_idx_3 = tc.cookie_idx
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req3, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Create case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req3, cmd_cookie, cmd, tc.naples)
tc.resp3 = api.Trigger(req3)
for command in tc.resp3.commands:
api.PrintCommandResults(command)
#
# Clear all sessions/flows
#
api.Trigger_TerminateAllCommands(resp_nc1)
req_nc2 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session"
add_naples_command(tc, req_nc2, cmd_cookie, cmd, tc.naples)
#
# Make sure that Client<=>Server Forwarding is set up
#
cmd_cookie = "trigger ping: Re-use case"
cmd = "ping -c1 %s -I %s" %\
(tc.server.ip_address, tc.client.interface)
add_command(tc, req_nc2, cmd_cookie, cmd, tc.client, False)
#
# Re-establish TCP-connection
#
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req_nc2, cmd_cookie, cmd, tc.naples)
cmd_cookie = "restart server"
cmd = "nc --listen %s" % (tc.server_port)
add_command(tc, req_nc2, cmd_cookie, cmd, tc.server, True)
cmd_cookie = "restart client"
cmd = "nc {} {} --source-port {} "\
.format(tc.server.ip_address, tc.server_port, tc.client_port)
add_command(tc, req_nc2, cmd_cookie, cmd, tc.client, True)
resp_nc2 = api.Trigger(req_nc2)
for command in resp_nc2.commands:
api.PrintCommandResults(command)
#
# Do applicable "show session" commands and
# retrieve Seq-num and Ack-num associated with the session
#
req4 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req4, cmd_cookie, cmd, tc.naples)
cmd_cookie = "show session detail"
cmd = "/nic/bin/halctl show session --dstport {} --dstip {} --yaml"\
.format(tc.server_port, tc.server.ip_address)
add_naples_command(tc, req4, cmd_cookie, cmd, tc.naples)
tc.resp4 = api.Trigger(req4)
cmd = tc.resp4.commands[-1]
for command in tc.resp4.commands:
api.PrintCommandResults(command)
tc.post_ctrckinf = get_conntrackinfo(cmd)
#
# Re-send Bad Data with TTL=0 from both Client and Server
#
req5 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd_cookie = "send bad data from Client TTL=0: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ttl 0 --data 10 {}".format(tc.client_port, tc.server_port,
tc.post_ctrckinf.r_tcpacknum,
tc.post_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TTL=0: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ttl 0 --data 10 {}".format(tc.server_port, tc.client_port,
tc.post_ctrckinf.i_tcpacknum,
tc.post_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.server, False)
#
# Re-send Bad Data with TCP-RSVD-FLAGS-BIT-0 set from both Client and Server
#
cmd_cookie = "send bad data from Client TCP-RSVD-FLAGS-BIT-0: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--xmas --data 10 {}".format(tc.client_port, tc.server_port,
tc.post_ctrckinf.r_tcpacknum + 10,
tc.post_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TCP-RSVD-FLAGS-BIT-0: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--xmas --data 10 {}".format(tc.server_port, tc.client_port,
tc.post_ctrckinf.i_tcpacknum + 10,
tc.post_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.server, False)
#
# Re-send Bad Data with TCP-RSVD-FLAGS-BIT-1 set from both Client and Server
#
cmd_cookie = "send bad data from Client TCP-RSVD-FLAGS-BIT-1: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ymas --data 10 {}".format(tc.client_port, tc.server_port,
tc.post_ctrckinf.r_tcpacknum + 20,
tc.post_ctrckinf.r_tcpseqnum,
tc.server.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.client, False)
cmd_cookie = "send bad data from Server TCP-RSVD-FLAGS-BIT-1: Re-use case"
cmd = "hping3 --count 1 --baseport {} --destport {} --setseq {} --setack {}\
--ymas --data 10 {}".format(tc.server_port, tc.client_port,
tc.post_ctrckinf.i_tcpacknum + 20,
tc.post_ctrckinf.i_tcpseqnum,
tc.client.ip_address)
add_command(tc, req5, cmd_cookie, cmd, tc.server, False)
tc.resp5 = api.Trigger(req5)
for command in tc.resp5.commands:
api.PrintCommandResults(command)
#
# Do "show session" command
#
req6 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cookie_idx_6 = tc.cookie_idx
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req6, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Re-use case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req6, cmd_cookie, cmd, tc.naples)
tc.resp6 = api.Trigger(req6)
for command in tc.resp6.commands:
api.PrintCommandResults(command)
#
# Do "show session" command after doing Sleep for 45secs
#
api.Trigger_TerminateAllCommands(resp_nc2)
req7 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cookie_idx_7 = tc.cookie_idx
cmd_cookie = "clear session"
cmd = "/nic/bin/halctl clear session; sleep 45"
add_naples_command(tc, req7, cmd_cookie, cmd, tc.naples)
cmd_cookie = "show session"
cmd = "/nic/bin/halctl show session"
add_naples_command(tc, req7, cmd_cookie, cmd, tc.naples)
#
# Trigger "metrics get IPv4FlowDropMetrics" output
#
cmd_cookie = "show flow-drop: Delete case"
cmd = "PATH=$PATH:/platform/bin/;\
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/platform/lib/:/nic/lib/;\
export PATH; export LD_LIBRARY_PATH;\
/nic/bin/delphictl metrics get IPv4FlowDropMetrics"
add_naples_command(tc, req7, cmd_cookie, cmd, tc.naples)
tc.resp7 = api.Trigger(req7)
for command in tc.resp7.commands:
api.PrintCommandResults(command)
return api.types.status.SUCCESS
def Trigger(tc):
pairs = api.GetLocalWorkloadPairs()
server = pairs[0][0]
client = pairs[0][1]
tc.cmd_cookies = []
naples = server
if not server.IsNaples():
naples = client
if not client.IsNaples():
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address, client.workload_name, client.ip_address)
api.Logger.info("Starting MSRPC test from %s" % (tc.cmd_descr))
api.Trigger_AddCommand(req,
server.node_name,
server.workload_name,
"tcpdump -nni %s > out.txt" % (server.interface),
background=True)
tc.cmd_cookies.append("tcpdump")
api.Trigger_AddCommand(req,
client.node_name,
client.workload_name,
"tcpdump -nni %s > out.txt" % (client.interface),
background=True)
tc.cmd_cookies.append("tcpdump")
api.Trigger_AddCommand(req,
server.node_name,
server.workload_name,
"sudo nc -l %s" % (MSRPC_PORT),
background=True)
tc.cmd_cookies.append("netcat start-server")
client_port = api.AllocateTcpPort()
api.Trigger_AddCommand(req,
client.node_name,
client.workload_name,
"sudo nc %s %s -p %s" %
(server.ip_address, MSRPC_PORT, client_port),
background=True)
tc.cmd_cookies.append("netcat start-client")
msrpcscript = dir_path + '/' + "msrpcscapy.py"
resp = api.CopyToWorkload(server.node_name, server.workload_name,
[msrpcscript], 'msrpcdir')
if resp is None:
return api.types.status.SUCCESS
resp = api.CopyToWorkload(client.node_name, client.workload_name,
[msrpcscript], 'msrpcdir')
if resp is None:
return api.types.status.SUCCESS
api.Trigger_AddNaplesCommand(
req, naples.node_name,
"/nic/bin/halctl show session --dstport 135 --yaml")
tc.cmd_cookies.append("show session yaml")
sesssetup = api.Trigger(req)
cmd = sesssetup.commands[-1]
api.PrintCommandResults(cmd)
tc.pre_ctrckinf = get_conntrackinfo(cmd)
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
msrpcpcap = dir_path + '/' + "msrpc_first_bind.pcap"
pkts = rdpcap(msrpcpcap)
clientidx = 0
serveridx = 0
client_ack = tc.pre_ctrckinf.i_tcpacknum
server_ack = tc.pre_ctrckinf.r_tcpacknum
filename = None
for pkt in pkts:
node = client.node_name
workload = client.workload_name
if pkt[IP].src == "172.31.9.1":
filename = ("msrpcscapy" + "%s" + ".pcap") % (clientidx)
msrpcscapy = dir_path + '/' + filename
pkt[Ether].src = client.mac_address
pkt[IP].src = client.ip_address
pkt[Ether].dst = server.mac_address
pkt[IP].dst = server.ip_address
pkt[TCP].sport = client_port
pkt[TCP].dport = 135
if clientidx == 0:
client_start_seq = pkt[TCP].seq
client_start_ack = pkt[TCP].ack
pkt[TCP].seq = tc.pre_ctrckinf.i_tcpseqnum + (pkt[TCP].seq -
client_start_seq)
pkt[TCP].ack = client_ack
server_ack = pkt[TCP].seq + len(pkt[TCP].payload) + 1
clientidx += 1
else:
filename = ("msrpcscapy" + "%s" + ".pcap") % (serveridx)
msrpcscapy = dir_path + '/' + filename
pkt[Ether].src = server.mac_address
pkt[IP].src = server.ip_address
pkt[Ether].dst = client.mac_address
pkt[IP].dst = client.ip_address
node = server.node_name
workload = server.workload_name
pkt[TCP].dport = client_port
pkt[TCP].sport = 135
if serveridx == 0:
server_start_seq = pkt[TCP].seq
server_start_ack = pkt[TCP].ack
pkt[TCP].seq = tc.pre_ctrckinf.r_tcpseqnum + (pkt[TCP].seq -
server_start_seq)
pkt[TCP].ack = server_ack
client_ack = pkt[TCP].seq + len(pkt[TCP].payload) + 1
serveridx += 1
del pkt[IP].chksum
del pkt[TCP].chksum
a = pkt.show(dump=True)
print(a)
wrpcap(msrpcscapy, pkt)
resp = api.CopyToWorkload(node, workload, [msrpcscapy], 'msrpcdir')
if resp is None:
continue
api.Trigger_AddCommand(
req, node, workload,
"sh -c 'cd msrpcdir && chmod +x msrpcscapy.py && ./msrpcscapy.py %s'"
% (filename))
tc.cmd_cookies.append("running #%s on node %s workload %s" %
(filename, node, workload))
os.remove(msrpcscapy)
api.Trigger_AddNaplesCommand(
req, naples.node_name,
"/nic/bin/halctl show session --dstport 135 --yaml")
tc.cmd_cookies.append("show session yaml")
firstbind = api.Trigger(req)
cmd = firstbind.commands[-1]
api.PrintCommandResults(cmd)
tc.post_ctrckinf = get_conntrackinfo(cmd)
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
msrpcpcap = dir_path + '/' + "msrpc_second_bind.pcap"
pkts = rdpcap(msrpcpcap)
client_ack = tc.post_ctrckinf.i_tcpacknum
server_ack = tc.post_ctrckinf.r_tcpacknum
clientidx = 0
serveridx = 0
filename = None
for pkt in pkts:
node = client.node_name
workload = client.workload_name
if pkt[IP].src == "172.31.9.1":
filename = ("msrpcscapy" + "%s" + ".pcap") % (clientidx)
msrpcscapy = dir_path + '/' + filename
pkt[Ether].src = client.mac_address
pkt[IP].src = client.ip_address
pkt[Ether].dst = server.mac_address
pkt[IP].dst = server.ip_address
pkt[TCP].sport = client_port
pkt[TCP].dport = 135
if clientidx == 0:
client_start_seq = pkt[TCP].seq
client_start_ack = pkt[TCP].ack
pkt[TCP].seq = tc.post_ctrckinf.i_tcpseqnum + (pkt[TCP].seq -
client_start_seq)
pkt[TCP].ack = client_ack
server_ack = pkt[TCP].seq + len(pkt[TCP].payload) + 1
clientidx += 1
else:
filename = ("msrpcscapy" + "%s" + ".pcap") % (serveridx)
msrpcscapy = dir_path + '/' + filename
pkt[Ether].src = server.mac_address
pkt[IP].src = server.ip_address
pkt[Ether].dst = client.mac_address
pkt[IP].dst = client.ip_address
node = server.node_name
workload = server.workload_name
pkt[TCP].dport = client_port
pkt[TCP].sport = 135
if serveridx == 0:
server_start_seq = pkt[TCP].seq
server_start_ack = pkt[TCP].ack
pkt[TCP].seq = tc.post_ctrckinf.r_tcpseqnum + (pkt[TCP].seq -
server_start_seq)
pkt[TCP].ack = server_ack
client_ack = pkt[TCP].seq + len(pkt[TCP].payload) + 1
serveridx += 1
del pkt[IP].chksum
del pkt[TCP].chksum
a = pkt.show(dump=True)
print(a)
wrpcap(msrpcscapy, pkt)
resp = api.CopyToWorkload(node, workload, [msrpcscapy], 'msrpcdir')
if resp is None:
continue
api.Trigger_AddCommand(
req, node, workload,
"sh -c 'cd msrpcdir && chmod +x msrpcscapy.py && ./msrpcscapy.py %s'"
% (filename))
tc.cmd_cookies.append("running #%s on node %s workload %s" %
(filename, node, workload))
os.remove(msrpcscapy)
api.Trigger_AddCommand(req,
server.node_name,
server.workload_name,
"sudo nc -l 49134",
background=True)
tc.cmd_cookies.append("msrpc start-server")
api.Trigger_AddCommand(req,
client.node_name,
client.workload_name,
"sudo nc %s 49134 -p 59374" % (server.ip_address),
background=True)
tc.cmd_cookies.append("msrpc start-client")
# Add Naples command validation
#api.Trigger_AddNaplesCommand(req, naples.node_name,
# "/nic/bin/halctl show security flow-gate | grep MSRPC")
#tc.cmd_cookies.append("show security flow-gate")
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show session")
tc.cmd_cookies.append("show session")
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp2 = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
term_resp = api.Trigger_TerminateAllCommands(firstbind)
tc.resp1 = api.Trigger_AggregateCommandsResponse(firstbind, term_resp)
term_resp = api.Trigger_TerminateAllCommands(sesssetup)
tc.resp = api.Trigger_AggregateCommandsResponse(sesssetup, term_resp)
#GetTcpdumpData(client)
#GetTcpdumpData(server)
return api.types.status.SUCCESS
def runIperfTest(tc, srv, cli):
if api.IsDryrun():
return api.types.status.SUCCESS
req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
req2 = api.Trigger_CreateExecuteCommandsRequest(serial=False)
proto = getattr(tc.iterators, "proto", 'tcp')
number_of_iperf_threads = getattr(tc.args, "iperfthreads", 1)
pktsize = getattr(tc.iterators, "pktsize", 512)
ipproto = getattr(tc.iterators, "ipproto", 'v4')
if ipproto == 'v6':
srv_ip_address = srv.ipv6_address
tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\
(srv.interface, srv.ipv6_address, cli.interface, cli.ipv6_address)
else:
srv_ip_address = srv.ip_address
tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\
(srv.interface, srv.ip_address, cli.interface, cli.ip_address)
api.Logger.info("Starting TSO test %s" % (tc.cmd_descr))
tc.srv_bad_csum = ionic_stats.getNetstatBadCsum(srv, proto)
tc.cli_bad_csum = ionic_stats.getNetstatBadCsum(cli, proto)
for i in range(number_of_iperf_threads):
if proto == 'tcp':
port = api.AllocateTcpPort()
else:
port = api.AllocateUdpPort()
file_name_suffix = "_instance" + \
str(i) + proto + "_" + ipproto + "_" + str(pktsize)
file_name = '/tmp/' + 'srv_' + srv.interface + file_name_suffix
iperf_server_cmd = cmd_builder.iperf_server_cmd(port=port)
api.Trigger_AddCommand(req1,
srv.node_name,
srv.workload_name,
iperf_server_cmd,
background=True)
file_name = '/tmp/' + 'cli_' + cli.interface + file_name_suffix
iperf_file_name = file_name + ".log"
iperf_client_cmd = cmd_builder.iperf_client_cmd(
server_ip=srv_ip_address,
port=port,
proto=proto,
pktsize=pktsize,
ipproto=ipproto)
# Once iperf JSON support is available, we don't need this hacks.
api.Trigger_AddCommand(
req2, cli.node_name, cli.workload_name,
iperf_client_cmd + " -J | tee " + iperf_file_name)
# Read the retransmission counter from the log
api.Trigger_AddCommand(
req2, cli.node_name, cli.workload_name, 'grep retrans ' +
iperf_file_name + '| tail -1| cut -d ":" -f 2 | cut -d "," -f 1')
# Read the bandwidth numbers.
api.Trigger_AddCommand(
req2, cli.node_name, cli.workload_name, 'cat ' + iperf_file_name +
' | grep bits_per_second | tail -1 | cut -d ":" -f 2 | cut -d "," -f 1'
)
# Read the bytes transferred numbers.
api.Trigger_AddCommand(
req2, cli.node_name, cli.workload_name, 'cat ' + iperf_file_name +
' | grep bytes | tail -1 | cut -d ":" -f 2 | cut -d "," -f 1')
trig_resp1 = api.Trigger(req1)
if trig_resp1 is None:
api.Logger.error("Failed to run iperf server")
return api.types.status.FAILURE
tc.resp = api.Trigger(req2)
if tc.resp is None:
api.Logger.error("Failed to run iperf client")
return api.types.status.FAILURE
for cmd in tc.resp.commands:
if cmd.exit_code != 0:
api.Logger.error("Failed to start client iperf\n")
api.PrintCommandResults(cmd)
return api.types.status.FAILURE
status, retran, bw = verifySingle(tc)
vlan = getattr(tc.iterators, 'vlantag', 'off')
vxlan = getattr(tc.iterators, 'vxlan', 'off')
tso = getattr(tc.iterators, 'tso_offload', 'off')
api.Logger.info(
"Result TSO: %s VLAN: %s VXLAN: %s Proto: %s/%s Pkt size: %d Threads: %d"
" Bandwidth: %d Mbps" % (tso, vlan, vxlan, proto, ipproto, pktsize,
number_of_iperf_threads, bw))
return status
def Trigger(tc):
#==============================================================
# trigger the commands
#==============================================================
krping_req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
dmesg_req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
ping_req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
# load krping on both nodes
tc.nodes = api.GetNaplesHostnames()
tc.os = api.GetNodeOs(tc.nodes[0])
for n in tc.nodes:
if tc.os == host.OS_TYPE_LINUX:
api.Trigger_AddHostCommand(
krping_req, n, "(lsmod | grep -w rdma_krping >/dev/null) || " +
"insmod {path}/krping/rdma_krping.ko".format(
path=tc.iota_path))
else:
api.Trigger_AddHostCommand(
krping_req, n, "(kldstat | grep -w krping >/dev/null) || " +
"kldload {path}/krping/krping.ko".format(path=tc.iota_path))
w1 = tc.w[0]
w2 = tc.w[1]
options = "port=9999,verbose,validate,size=65536,"
# cmd for server
if api.GetNodeOs(w1.node_name) == host.OS_TYPE_LINUX:
krpfile = "/proc/krping"
else:
krpfile = "/dev/krping"
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" % \
(w1.workload_name, w1.ip_address, \
w2.workload_name, w2.ip_address)
api.Logger.info("Starting krping_rdma test from %s" % (tc.cmd_descr))
# If the client fails to connect or otherwise misbehaves, IOTA
# can get stuck with a server waiting forever. This causes
# module unload to fail and cascading failures after that.
# Tell the server to only wait this long before giving up.
server_options = options + "wait=30,"
cmd = "sudo echo -n 'server,addr={addr},{opstr}' > {kfile}".format(
addr=w1.ip_address, opstr=server_options, kfile=krpfile)
api.Trigger_AddCommand(krping_req,
w1.node_name,
w1.workload_name,
cmd,
background=True)
# It takes a few seconds before the server starts listening.
cmd = 'sleep 3'
api.Trigger_AddCommand(krping_req, w1.node_name, w1.workload_name, cmd)
cmd = "sudo echo -n 'client,addr={addr},{opstr}' > {kfile}".format(
addr=w1.ip_address, opstr=options, kfile=krpfile)
api.Trigger_AddCommand(krping_req,
w2.node_name,
w2.workload_name,
cmd,
background=True)
krping_trig_resp = api.Trigger(krping_req)
#Run RDMA while krping is running
rdma_perf.Trigger(tc)
krping_term_resp = api.Trigger_TerminateAllCommands(krping_trig_resp)
tc.krping_resp = api.Trigger_AggregateCommandsResponse(
krping_trig_resp, krping_term_resp)
# dmesg commands
api.Trigger_AddCommand(dmesg_req, w1.node_name, w1.workload_name,
"dmesg | tail -20 | grep rdma-ping-")
api.Trigger_AddCommand(dmesg_req, w2.node_name, w2.workload_name,
"dmesg | tail -20 | grep rdma-ping-")
dmesg_trig_resp = api.Trigger(dmesg_req)
dmesg_term_resp = api.Trigger_TerminateAllCommands(dmesg_trig_resp)
tc.dmesg_resp = api.Trigger_AggregateCommandsResponse(
dmesg_trig_resp, dmesg_term_resp)
# ping to check if the pipeline is stuck
api.Trigger_AddCommand(ping_req, w1.node_name, w1.workload_name,
"ping -c 10 -t 10 " + w2.ip_address)
api.Trigger_AddCommand(ping_req, w2.node_name, w2.workload_name,
"ping -c 10 -t 10 " + w1.ip_address)
ping_trig_resp = api.Trigger(ping_req)
ping_term_resp = api.Trigger_TerminateAllCommands(ping_trig_resp)
tc.ping_resp = api.Trigger_AggregateCommandsResponse(
ping_trig_resp, ping_term_resp)
return api.types.status.SUCCESS
def Verify(tc):
if tc.skip:
return api.types.status.SUCCESS
if tc.pktloss_verify:
if tc.background and tc.bg_cmd_resp is None:
api.Logger.error("Failed in background Ping cmd trigger")
return api.types.status.FAILURE
if tc.resp is None:
api.Logger.error("Received empty response for config request")
return api.types.status.FAILURE
else:
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Rollout request failed")
return api.types.status.FAILURE
# wait for upgrade to complete. status can be found from the presence of /update/pds_upg_status.txt
api.Logger.info("Sleep for 70 secs before checking for /update/pds_upg_status.txt")
misc_utils.Sleep(70)
status_in_progress = True
while status_in_progress:
misc_utils.Sleep(1)
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(req, node, "grep -v in-progress /update/pds_upg_status.txt", timeout=2)
api.Logger.info("Checking for status not in-progress in file /update/pds_upg_status.txt")
resp = api.Trigger(req)
status_in_progress = False
for cmd_resp in resp.commands:
#api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
status_in_progress = True
#api.Logger.info("File /update/pds_upg_status.txt not found")
else:
api.Logger.info("Status other than in-progress found in /update/pds_upg_status.txt")
# # push interface config updates after upgrade completes
# UpdateConfigAfterUpgrade(tc)
for i in range(10):
api.Logger.info("Sending ARPing, retry count %s"%i)
# Send Grat Arp for learning
arping.SendGratArp(tc.wloads)
misc_utils.Sleep(1)
result = CheckRolloutStatus(tc)
# ensure connectivity after upgrade
if VerifyConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check Post Upgrade.")
result = api.types.status.FAILURE
error_str = None
if tc.pktloss_verify:
# If rollout status is failure, then no need to wait for traffic test
if result == api.types.status.SUCCESS:
api.Logger.info("Sleep for %s secs for traffic test to complete"%tc.sleep)
misc_utils.Sleep(tc.sleep)
pkt_loss_duration = 0
# terminate background traffic and calculate packet loss duration
if tc.background:
if ping.TestTerminateBackgroundPing(tc, tc.pktsize,\
pktlossverif=tc.pktlossverif) != api.types.status.SUCCESS:
api.Logger.error("Failed in Ping background command termination.")
result = api.types.status.FAILURE
# calculate max packet loss duration for background ping
pkt_loss_duration = ping.GetMaxPktLossDuration(tc, interval=tc.interval)
if pkt_loss_duration != 0:
indent = "-" * 10
if tc.pktlossverif:
result = api.types.status.FAILURE
api.Logger.error(f"{indent} Packet Loss duration during UPGRADE of {tc.Nodes} is {pkt_loss_duration} secs {indent}")
if tc.allowed_down_time and (pkt_loss_duration > tc.allowed_down_time):
api.Logger.error(f"{indent} Exceeded allowed Loss Duration {tc.allowed_down_time} secs {indent}")
result = api.types.status.FAILURE
else:
api.Logger.info("No Packet Loss Found during UPGRADE Test")
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
result = api.types.status.FAILURE
return result
def Trigger(tc):
# check that there have been packets showing up after previous tests
# ip -s link show <pf>
packet_count = 1000
mode = tc.iterators.mode
if mode == "RX":
pkt_cnt_before = GetVFRxStats(tc.host1, tc.pf_1, tc.vfid)
else:
pkt_cnt_before = GetVFTxStats(tc.host1, tc.pf_1, tc.vfid)
if pkt_cnt_before == -1:
api.Logger.error("Getting VF %s stats failed" % mode)
return api.types.status.ERROR
reverse = True if mode == "RX" else False
servercmd = iperf.ServerCmd(server_ip=tc.remote_ip, port=7777)
clientcmd = iperf.ClientCmd(tc.remote_ip,
packet_count=packet_count,
client_ip=tc.vf_ip,
jsonOut=True,
port=7777,
proto='udp',
reverse=reverse)
sreq = api.Trigger_CreateExecuteCommandsRequest(serial=True)
creq = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddHostCommand(sreq, tc.host2, servercmd, background=True)
api.Trigger_AddHostCommand(creq, tc.host1, clientcmd, timeout=30)
server_resp = api.Trigger(sreq)
if not server_resp:
api.Logger.error("Unable to execute server command")
return api.types.status.ERROR
time.sleep(5)
client_resp = api.Trigger(creq)
if not client_resp:
api.Logger.error("Unable to execute client command")
return api.types.status.ERROR
resp = client_resp.commands.pop()
if resp.exit_code != 0:
api.Logger.error("Iperf client failed with exit code %d" %
resp.exit_code)
api.PrintCommandResults(resp)
return api.types.status.ERROR
if not iperf.Success(resp.stdout):
api.Logger.error("Iperf failed with error: %s" %
iperf.Error(resp.stdout))
return api.types.status.ERROR
if mode == "RX":
pkt_cnt_after = GetVFRxStats(tc.host1, tc.pf_1, tc.vfid)
else:
pkt_cnt_after = GetVFTxStats(tc.host1, tc.pf_1, tc.vfid)
delta_pkt_cnt = pkt_cnt_after - pkt_cnt_before
if packet_count > delta_pkt_cnt:
api.Logger.error("Incorrect %s stats, expected %d observed %d" %
(mode, packet_count, delta_pkt_cnt))
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Trigger(tc):
triplet = GetThreeWorkloads()
server = triplet[0][0]
client1 = triplet[0][1]
client2 = triplet[0][2]
tc.cmd_cookies = []
naples = server
if not server.IsNaples():
naples = client1
if not client1.IsNaples():
naples = client2
if not client2.IsNaples():
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\
(server.workload_name, server.ip_address, client1.workload_name, client1.ip_address)
api.Logger.info("Starting SUNRPC test from %s" % (tc.cmd_descr))
SetupNFSServer(server, client1)
api.Trigger_AddCommand(
req, server.node_name, server.workload_name,
"sh -c 'ls -al /home/sunrpcmntdir | sudo grep sunrpc_file.txt'")
tc.cmd_cookies.append("Before rpc")
api.Trigger_AddCommand(
req, client1.node_name, client1.workload_name,
"sudo sh -c 'mkdir -p /home/sunrpcdir && sudo mount %s:/home/sunrpcmntdir /home/sunrpcdir' "
% (server.ip_address))
tc.cmd_cookies.append("Create mount point")
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"sudo chmod 777 /home/sunrpcdir")
tc.cmd_cookies.append("add permission")
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"mv sunrpcdir/sunrpc_file.txt /home/sunrpcdir/")
tc.cmd_cookies.append("Create file")
api.Trigger_AddCommand(req, client1.node_name, client1.workload_name,
"ls -al /home/sunrpcdir")
tc.cmd_cookies.append("verify file")
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
"ls -al /home/sunrpcmntdir/")
tc.cmd_cookies.append("After rpc")
api.Trigger_AddCommand(req, server.node_name, server.workload_name,
"sh -c 'cat /home/sunrpcmntdir/sunrpc_file.txt'")
tc.cmd_cookies.append("After rpc")
# Add Naples command validation
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show session --alg sun_rpc")
tc.cmd_cookies.append("show session")
api.Trigger_AddNaplesCommand(req, naples.node_name,
"/nic/bin/halctl show nwsec flow-gate")
tc.cmd_cookies.append("show security flow-gate")
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
# Get it from flow gate
dport = 2049
req2 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddCommand(
req2, server.node_name, server.workload_name,
"hping3 -c 1 -s 45535 -p {} -d 0 -S {}".format(dport,
client1.ip_address))
tc.cmd_cookies.append("Hping from different direction")
# Get the timeout from the config
api.Trigger_AddNaplesCommand(
req2, naples.node_name,
"/nic/bin/halctl show session --srcip %s | grep SYN" %
(client1.ip_address))
tc.cmd_cookies.append("show session reverse direction")
trig_resp2 = api.Trigger(req2)
term_resp2 = api.Trigger_TerminateAllCommands(trig_resp2)
tc.resp2 = api.Trigger_AggregateCommandsResponse(trig_resp2, term_resp2)
CleanupNFSServer(server, client1)
return api.types.status.SUCCESS
def send_pkt(tc, node, pkt_gen, flow, pkt_cnt):
# ==========================================
# Send and Receive packets in H2S direction
# ==========================================
pkt_gen.set_dir_('h2s')
pkt_gen.set_sip(flow.sip)
pkt_gen.set_dip(flow.dip)
pkt_gen.set_nat_flows_h2s_tx(nat_flows_h2s_tx)
pkt_gen.set_nat_flows_h2s_rx(nat_flows_h2s_rx)
if flow.proto == 'UDP' or flow.proto == 'TCP':
pkt_gen.set_sport(flow.sport)
pkt_gen.set_dport(flow.dport)
h2s_req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
# ==========
# Rx Packet
# ==========
pkt_gen.set_encap(True)
pkt_gen.set_Rx(True)
pkt_gen.set_vlan(tc.up0_vlan)
pkt_gen.set_smac(tc.up1_mac)
pkt_gen.set_dmac(tc.up0_mac)
pkt_gen.setup_pkt()
recv_cmd = "./recv_pkt.py --intf_name %s --pcap_fname %s "\
"--timeout %s --pkt_cnt %d" % (tc.up0_intf,
pktgen.DEFAULT_H2S_RECV_PKT_FILENAME,
str(SNIFF_TIMEOUT), pkt_cnt)
api.Trigger_AddHostCommand(h2s_req, node.Name(), recv_cmd, background=True)
# ==========
# Tx Packet
# ==========
pkt_gen.set_encap(False)
pkt_gen.set_Rx(False)
pkt_gen.set_smac(tc.up1_mac)
pkt_gen.set_dmac(tc.up0_mac)
pkt_gen.set_vlan(tc.up1_vlan)
pkt_gen.setup_pkt()
send_cmd = "./send_pkt.py --intf_name %s --pcap_fname %s "\
"--pkt_cnt %d" % (tc.up1_intf,
pktgen.DEFAULT_H2S_GEN_PKT_FILENAME, pkt_cnt)
api.Trigger_AddHostCommand(h2s_req, node.Name(), 'sleep 0.5')
api.Trigger_AddHostCommand(h2s_req, node.Name(), send_cmd)
trig_resp = api.Trigger(h2s_req)
time.sleep(SNIFF_TIMEOUT)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
h2s_resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
tc.resp.append(h2s_resp)
# ==========================================
# Send and Receive packets in S2H direction
# ==========================================
pkt_gen.set_dir_('s2h')
pkt_gen.set_sip(flow.dip)
pkt_gen.set_dip(flow.sip)
pkt_gen.set_nat_flows_s2h_tx(nat_flows_s2h_tx)
pkt_gen.set_nat_flows_s2h_rx(nat_flows_s2h_rx)
if flow.proto == 'UDP' or flow.proto == 'TCP':
pkt_gen.set_sport(flow.dport)
pkt_gen.set_dport(flow.sport)
s2h_req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
# ==========
# Rx Packet
# ==========
pkt_gen.set_encap(False)
pkt_gen.set_Rx(True)
pkt_gen.set_smac(tc.up0_mac)
pkt_gen.set_dmac(tc.up1_mac)
pkt_gen.set_vlan(tc.up1_vlan)
pkt_gen.setup_pkt()
recv_cmd = "./recv_pkt.py --intf_name %s --pcap_fname %s "\
"--timeout %s --pkt_cnt %d" % (tc.up1_intf,
pktgen.DEFAULT_S2H_RECV_PKT_FILENAME,
str(SNIFF_TIMEOUT), pkt_cnt)
api.Trigger_AddHostCommand(s2h_req, node.Name(), recv_cmd, background=True)
# ==========
# Tx Packet
# ==========
pkt_gen.set_encap(True)
pkt_gen.set_Rx(False)
pkt_gen.set_smac(tc.up0_mac)
pkt_gen.set_dmac(tc.up1_mac)
pkt_gen.set_vlan(tc.up0_vlan)
pkt_gen.setup_pkt()
send_cmd = "./send_pkt.py --intf_name %s --pcap_fname %s "\
"--pkt_cnt %d" % (tc.up0_intf,
pktgen.DEFAULT_S2H_GEN_PKT_FILENAME, pkt_cnt)
api.Trigger_AddHostCommand(s2h_req, node.Name(), 'sleep 0.5')
api.Trigger_AddHostCommand(s2h_req, node.Name(), send_cmd)
trig_resp = api.Trigger(s2h_req)
time.sleep(SNIFF_TIMEOUT)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
s2h_resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
tc.resp.append(s2h_resp)