def Verify(tc):
api.Logger.info("Verify.")
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
if tc.resp == None:
return api.types.status.SUCCESS
for cmd in tc.resp.commands:
if tc.cmd_cookies['show after'] == cmd.command:
if not cmd.stdout:
return api.types.status.SUCCESS
print(cmd.stdout)
yaml_out = get_yaml(cmd)
if tc.resp_flow:
flow = get_respflow(yaml_out)
else:
flow = get_initflow(yaml_out)
state = get_tcpstate(flow)
if int(state) == TcpState.ESTABLISHED:
return api.types.status.SUCCESS
else:
return api.types.status.FAILURE
return api.types.status.FAILURE
def Verify(tc):
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
if tc.skip: return api.types.status.SUCCESS
if tc.resp is None:
return api.types.status.FAILURE
result = api.types.status.SUCCESS
cookie_idx = 0
for cmd in tc.resp.commands:
api.Logger.info("Ping Results for %s" % (tc.cmd_cookies[cookie_idx]))
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
if api.GetConfigNicMode() == 'hostpin' and tc.iterators.pktsize > 1024:
result = api.types.status.SUCCESS
else:
try:
found = re.search(cmd.node_name+"_(.*)_subif_(.*$)",cmd.entity_name)
if found:
debugPing(cmd, found.group(1))
else:
api.Logger.debug("failed to determine interface name from {0}".format(cmd.entity_name))
except:
api.Logger.debug("failed to debug {0} ping issue".format(cmd.entity_name))
result = api.types.status.FAILURE
#for future use
#elif tc.args.type == 'local_only':
# return api.types.status.CRITICAL
#else:
# raise OfflineTestbedException
cookie_idx += 1
return result
def Verify(tc):
api.Logger.info("Verify.")
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
if tc.resp is None:
api.Logger.info("Null response from aggregartecommands")
return api.types.status.FAILURE
cookie_idx = 0
for cmd in tc.resp.commands:
api.Logger.info("Results for %s" % (tc.cmd_cookies[cookie_idx]))
api.PrintCommandResults(cmd)
metrics = utils.GetDelphiSessionSummaryMetrics(tc.wc_server.node_name)
api.Logger.info("After Session summary metrics for %s => %s" % \
(tc.wc_server.node_name, metrics))
limit = utils.GetSessionLimit(tc.proto)
if limit == api.types.status.FAILURE or limit == 0:
api.Logger.error("Session Limit invalid : %d" % limit)
return api.types.status.FAILURE
if metrics[tc.metric_field] != limit:
api.Logger.error("%s : %d Expected : %d" % \
(tc.metric_field, metrics[tc.metric_field], limit))
return api.types.status.FAILURE
api.Logger.info("%s session limit metric verification Success" % tc.proto)
return VerifySessionEvents(tc)
def Trigger(tc):
api.Logger.info("Trigger.")
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if tc.resp_flow:
new_seq_num = tc.pre_ctrckinf.r_tcpseqnum + tc.pre_ctrckinf.i_tcpwinsz * (
2**tc.pre_ctrckinf.i_tcpwinscale)
cmd_cookie = "hping3 -c 1 -s 1237 -p 52255 -M {} -L {} --rst --ack --tcp-timestamp {}".format(
new_seq_num, tc.pre_ctrckinf.r_tcpacknum, tc.client.ip_address)
add_command(req, tc, 'fail ping', tc.server, cmd_cookie)
else:
new_seq_num = tc.pre_ctrckinf.i_tcpseqnum + tc.pre_ctrckinf.r_tcpwinsz * (
2**tc.pre_ctrckinf.r_tcpwinscale)
cmd_cookie = "hping3 -c 1 -s 52255 -p 1237 -M {} -L {} --rst --ack --tcp-timestamp {}".format(
new_seq_num, tc.pre_ctrckinf.i_tcpacknum, tc.server.ip_address)
add_command(req, tc, 'fail ping', tc.client, cmd_cookie)
cmd_cookie = "sleep 3 && /nic/bin/halctl show session --dstport 1237 --dstip {} --yaml".format(
tc.server.ip_address)
add_command(req, tc, 'show after', tc.client, cmd_cookie, naples=True)
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.client, cmd_cookie, naples=True)
if tc.server.IsNaples():
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.server, cmd_cookie, naples=True)
trig_resp = api.Trigger(req)
term_resp1 = api.Trigger_TerminateAllCommands(tc.setup_cmd_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp1)
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
return api.types.status.SUCCESS
def Verify(tc):
api.Logger.info("Verify.")
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
if tc.resp == None:
return api.types.status.SUCCESS
for cmd in tc.resp.commands:
if tc.cmd_cookies['show after'] == cmd.command:
if not cmd.stdout:
return api.types.status.SUCCESS
print(cmd.stdout)
yaml_out = get_yaml(cmd)
if tc.resp_flow:
flow = get_respflow(yaml_out)
else:
flow = get_initflow(yaml_out)
conn_info = get_conntrack_info(flow)
excep = get_exceptions(conn_info)
if (excep['tcpoutofwindow'] == False):
return api.types.status.FAILURE
return api.types.status.SUCCESS
#print(tc.resp)
return api.types.status.FAILURE
def Verify(tc):
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
try:
if tc.skip: return api.types.status.SUCCESS
cstat = tc.clientHandle.get_stats()
sstat = tc.serverHandle.get_stats()
ct = cstat['traffic']['client']
st = sstat['traffic']['server']
cleanup(tc)
if tc.iterators.proto == 'tcp':
return TRexIotaWrapper.validateTcpStats(ct,
st,
int(tc.iterators.cps),
int(tc.iterators.duration),
tolerance=tc.args.drop_tol)
elif tc.iterators.proto == 'udp':
return TRexIotaWrapper.validateUdpStats(ct,
st,
int(tc.iterators.cps),
int(tc.iterators.duration),
tolerance=tc.args.drop_tol)
else:
raise Exception("Unknown protocol %s" % tc.iterators.proto)
except Exception as e:
traceback.print_exc()
api.Logger.error("Failed to verify the CPS : %s" % (e))
cleanup(tc)
return api.types.status.FAILURE
def Trigger(tc):
api.Logger.info("Trigger.")
if tc.skip: return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if tc.resp_flow:
cmd_cookie = 'tcpdump -nni {} "src host {} and dst host {} and src port 1237"'.format(
tc.client.interface, tc.server.ip_address, tc.client.ip_address)
print(cmd_cookie)
add_command(req, tc, 'tcpdump', tc.client, cmd_cookie, True)
new_seq_num = tc.pre_ctrckinf.r_tcpseqnum + tc.pre_ctrckinf.i_tcpwinsz * (
2**tc.pre_ctrckinf.i_tcpwinscale)
#left out of window - retransmit
cmd_cookie = "hping3 -c 1 -s 1237 -p 52255 -M {} -L {} --ack {} -d 900 --tcp-timestamp".format(
wrap_around(tc.pre_ctrckinf.r_tcpseqnum, 0),
tc.pre_ctrckinf.r_tcpacknum, tc.client.ip_address)
add_command(req, tc, 'fail ping', tc.server, cmd_cookie)
else:
cmd_cookie = 'tcpdump -nni {} "src host {} and dst host {} and src port 1237"'.format(
tc.client.interface, tc.client.ip_address, tc.server.ip_address)
print(cmd_cookie)
add_command(req, tc, 'tcpdump', tc.client, cmd_cookie, True)
new_seq_num = tc.pre_ctrckinf.i_tcpseqnum + tc.pre_ctrckinf.r_tcpwinsz * (
2**tc.pre_ctrckinf.r_tcpwinscale)
#left out of window - retransmit
cmd_cookie = "hping3 -c 1 -s 52255 -p 1237 -M {} -L {} --ack --tcp-timestamp {} -d 10".format(
wrap_around(tc.pre_ctrckinf.i_tcpseqnum, 0),
tc.pre_ctrckinf.i_tcpacknum, tc.server.ip_address)
add_command(req, tc, "fail ping", tc.client, cmd_cookie)
cmd_cookie = "sleep 3 && /nic/bin/halctl show session --dstport 1237 --dstip {} --yaml".format(
tc.server.ip_address)
add_command(req, tc, 'show after', tc.client, cmd_cookie, naples=True)
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.client, cmd_cookie, naples=True)
if tc.server.IsNaples():
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.server, cmd_cookie, naples=True)
sec_resp = api.Trigger(req)
term_first_resp = api.Trigger_TerminateAllCommands(tc.first_resp)
term_sec_resp = api.Trigger_TerminateAllCommands(sec_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(tc.first_resp,
term_first_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(sec_resp, term_sec_resp)
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
return api.types.status.SUCCESS
def Verify(tc):
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
api.Logger.info("Validating ...")
if tc.resp is None:
return api.types.status.FAILURE
result = api.types.status.SUCCESS
return result
def Verify(tc):
result = api.types.status.SUCCESS
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
try:
# Disconnect and stop Trex, so that we dont get any more packets from Trex.
# Without this session verification will fail.
cleanup(tc)
ret = verifySessions(tc)
if ret != api.types.status.SUCCESS:
return ret
api.Logger.info("Verified session stats successfully.")
if tc.mem_leak_test:
api.Logger.info(
"Waiting for 300 secs to allow system to reclaim "
"free memory from heap before collecting final MemStats")
# wait for 300 secs for memory to be freed to heap
time.sleep(300)
# Reclaim all free memory from heap at end of test
if enable_memory_trim:
execHalMemTrimCmd(tc)
memStatsObjClient.CollectMemUsageSnapshot(log=True)
memStatsObjClient.PrintMemUsageHistory()
if memStatsObjClient.CheckMemLeakFromHistory(
pname='hal', acceptable_increase=tc.mem_incr_threshold):
api.Logger.error("MemLeak Found...")
result = api.types.status.FAILURE
else:
api.Logger.info("NO MemLeak Found in HAL process...")
ret = utils.isHalAlive()
if ret != api.types.status.SUCCESS:
api.Logger.error("HAL is DEAD ..")
result = api.types.status.FAILURE
else:
api.Logger.info("Verified the HAL process.")
except Exception as e:
traceback.print_exc()
api.Logger.error("Failed to verify : %s" % (e))
return api.types.status.FAILURE
return result
def Trigger(tc):
api.Logger.info("Trigger.")
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
if tc.resp_flow:
#right of window overlap
iwindo_size = tc.pre_ctrckinf.i_tcpwinsz * (
2**tc.pre_ctrckinf.i_tcpwinscale)
cmd_cookie = "hping3 -c 1 -s {} -p {} -M {} -L {} --fin --ack --tcp-timestamp {} -d {}".format(
tc.server_port, tc.client_port,
wrap_around(tc.pre_ctrckinf.r_tcpseqnum + iwindo_size, -200),
tc.pre_ctrckinf.r_tcpacknum, tc.client.ip_address, 500)
add_command(req, tc, 'fail ping', tc.server, cmd_cookie)
else:
rwindo_size = tc.pre_ctrckinf.r_tcpwinsz * (
2**tc.pre_ctrckinf.r_tcpwinscale)
#right of window overlap
cmd_cookie = "hping3 -c 1 -s {} -p {} -M {} -L {} --fin --ack --tcp-timestamp {} -d {}".format(
tc.client_port, tc.server_port,
wrap_around(tc.pre_ctrckinf.i_tcpseqnum + rwindo_size, -200),
tc.pre_ctrckinf.i_tcpacknum, tc.server.ip_address, 500)
add_command(req, tc, 'fail ping', tc.client, cmd_cookie)
cmd_cookie = "/nic/bin/halctl show session --dstport {} --dstip {} --yaml".format(
tc.server_port, tc.server.ip_address)
add_command(req, tc, 'show after', tc.client, cmd_cookie, naples=True)
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.client, cmd_cookie, naples=True)
if tc.server.IsNaples():
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.server, cmd_cookie, naples=True)
trig_resp = api.Trigger(req)
term_resp1 = api.Trigger_TerminateAllCommands(tc.setup_cmd_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp1)
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
return api.types.status.SUCCESS
def Verify(tc):
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
if tc.resp == None:
return api.types.status.SUCCESS
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if not tc.pcap_file_name:
api.Logger.error("Invalid Pcap file")
return api.types.status.FAILURE
ret = api.CopyFromWorkload(tc.client.node_name, tc.client.workload_name,
[tc.pcap_file_name], tc.dir_path)
if not ret:
return api.types.status.FAILURE
return __verify_pcap_packet_data_len(f"{tc.dir_path}/{tc.pcap_file_name}",
tc.test_mss)
def Trigger(tc):
api.Logger.info("Trigger.")
new_seq_num = tc.pre_ctrckinf.i_tcpseqnum + tc.pre_ctrckinf.r_tcpwinsz * (
2**tc.pre_ctrckinf.r_tcpwinscale)
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
#within the window - outoforder
cmd_cookie = "sleep 2 && hping3 -c 1 -s 52252 -p 1234 -M {} -L {} --ack --tcp-timestamp {} -d 10".format(
tc.pre_ctrckinf.i_tcpseqnum + 100, tc.pre_ctrckinf.i_tcpacknum,
tc.server.ip_address)
api.Trigger_AddCommand(req, tc.client.node_name, tc.client.workload_name,
cmd_cookie)
tc.cmd_cookies['fail ping'] = cmd_cookie
cmd_cookie = "sleep 3 && /nic/bin/halctl show session --dstport 1234 --dstip {} --yaml".format(
tc.server.ip_address)
api.Trigger_AddNaplesCommand(req, tc.client.node_name, cmd_cookie)
tc.cmd_cookies['show after'] = cmd_cookie
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.client, cmd_cookie, naples=True)
if tc.server.IsNaples():
cmd_cookie = "/nic/bin/halctl clear session"
add_command(req, tc, 'clear', tc.server, cmd_cookie, naples=True)
trig_resp = api.Trigger(req)
term_resp = api.Trigger_TerminateAllCommands(trig_resp)
term_resp1 = api.Trigger_TerminateAllCommands(tc.setup_cmd_resp)
tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp)
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
return api.types.status.SUCCESS
def Verify(tc):
if getattr(tc.args, 'vmotion_enable', False):
vmotion_utils.PrepareWorkloadRestore(tc)
return tc.ret