def Trigger(tc):
# skip some iterator cases
if skip_curr_test(tc):
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest()
trig_cmd1 = "cd %s; tar -czvf %s output" % (DTS_OUTPUT_PATH,
DTS_LOG_TAR_FILE)
api.Trigger_AddHostCommand(req,
tc.tester_node.Name(),
trig_cmd1,
timeout=60)
trig_resp = api.Trigger(req)
tc.resp.append(trig_resp)
tc_dir = tc.GetLogsDir()
api.Logger.info("Dest dir is %s: Src is %s" %
(tc_dir, DTS_OUTPUT_PATH + DTS_LOG_TAR_FILE))
resp = api.CopyFromHost(tc.tester_node_name,
[DTS_OUTPUT_PATH + DTS_LOG_TAR_FILE], tc_dir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy dts log file from node: %s" %
tc.tester_node_name)
result = api.types.status.FAILURE
return api.types.status.SUCCESS
def stop_single_pcap_capture(tc):
if not getattr(tc, "pcap_trigger", None):
return api.types.status.SUCCESS
api.Trigger_TerminateAllCommands(tc.pcap_trigger)
nodes = api.GetWorkloadNodeHostnames()
tc_dir = tc.GetLogsDir()
resp = api.CopyFromHost(n, [tc.pcap_filename], tc_dir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy from to Node: %s" % n)
return api.types.status.FAILURE
return api.types.status.SUCCESS
def CollectTechSupport(tsName):
try:
#global __CURREN_TECHSUPPORT_CNT
#__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1
if GlobalOptions.pipeline in ["apulu"]:
return types.status.SUCCESS
Logger.info("Collecting techsupport for testsuite {0}".format(tsName))
tsName = re.sub('\W', '_', tsName)
logDir = GlobalOptions.logdir
if not logDir.endswith('/'):
logDir += '/'
logDir += 'techsupport/'
if not os.path.exists(logDir):
os.mkdir(logDir)
nodes = api.GetNaplesHostnames()
req = api.Trigger_CreateExecuteCommandsRequest()
for n in nodes:
Logger.info("Techsupport for node: %s" % n)
common.AddPenctlCommand(
req, n, "system tech-support -b %s-tech-support" % (n))
resp = api.Trigger(req)
result = types.status.SUCCESS
for n, cmd in zip(nodes, resp.commands):
#api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
Logger.error(
"Failed to execute penctl system tech-support on node: %s. err: %d"
% (n, cmd.exit_code))
result = types.status.FAILURE
continue
# Copy tech support tar out
# TAR files are created at: pensando/iota/entities/node1_host/<test_case>
ntsn = "%s-tech-support.tar.gz" % (n)
resp = api.CopyFromHost(n, [ntsn], logDir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
Logger.error(
"Failed to copy techsupport file %s from node: %s" %
(ntsn, n))
result = types.status.FAILURE
continue
os.rename(logDir + ntsn, logDir + tsName + '_' + ntsn)
#if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN:
# return types.status.CRITICAL
return result
except AttributeError:
Logger.debug('failed to collect tech support. node list not setup yet')
except:
Logger.debug('failed to collect tech support. error was: {0}'.format(
traceback.format_exc()))
return types.status.CRITICAL
def Verify(tc):
for cmd in tc.install_cmd_resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd):
api.Logger.error("Installation failed on Node %s\n" %
(cmd.node_name))
return api.types.status.FAILURE
for cmd in tc.deploy_cmd_resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd):
api.Logger.error("Deploy failed on Node %s\n" % (cmd.node_name))
return api.types.status.FAILURE
for cmd in tc.validate_cmd_resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd):
api.Logger.error("Validation procedure failed on Node %s\n" %
(cmd.node_name))
return api.types.status.FAILURE
dir_path = os.path.dirname(os.path.realpath(__file__))
api.Logger.info("Dir Path is: %s\n" % (dir_path))
for n in tc.nodes:
api.Logger.info("Getting data for: %s\n" % (n))
resp = api.CopyFromHost(n,
["pencap/linux/techsupport/techsupport.json"],
dir_path)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy client file: %s\n" % (n))
return api.types.status.FAILURE
jdata = {}
try:
with open(dir_path + "/techsupport.json") as f:
jdata = json.load(f)
except:
api.Logger.error("Unable to open local file from node: %s\n" % (n))
return api.types.status.FAILURE
for data in jdata:
array_of_dumps = jdata[data]
for dump in array_of_dumps:
dump_menu_title = ""
for item in dump:
if (isinstance(dump[item], str) == True):
dump_menu_title += dump[item] + "_"
api.Logger.info("Dumped data: %s\n" % (dump_menu_title))
return api.types.status.SUCCESS
def stop_pcap_capture(tc):
api.Trigger_TerminateAllCommands(tc.pcap_trigger)
nodes = api.GetWorkloadNodeHostnames()
tc_dir = tc.GetLogsDir()
for n, host_intfs in tc.host_intfs.items():
if len(host_intfs) == 0:
api.Logger.error("No host interfaces for node :%s" % n)
return api.types.status.FAILURE
for intfObj in host_intfs:
intf = intfObj.Name()
resp = api.CopyFromHost(n, [pcap_file_name(intf)], tc_dir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy from to Node: %s" % n)
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Main(tc):
if tc.GetStatus() == api.types.status.SUCCESS:
return api.types.status.SUCCESS
global __CURREN_TECHSUPPORT_CNT
__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1
api.Logger.info("TC: %s failed. Collecting techsupport." % (tc.Name()))
nodes = api.GetNaplesHostnames()
req = api.Trigger_CreateExecuteCommandsRequest()
tc_dir = tc.GetLogsDir()
for n in nodes:
api.Logger.info("Techsupport for node: %s" % n)
common.AddPenctlCommand(req, n,
"system tech-support -b %s-tech-support" % (n))
tc.resp = api.Trigger(req)
result = api.types.status.SUCCESS
for n, cmd in zip(nodes, tc.resp.commands):
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error(
"Failed to execute penctl system tech-support on node: %s. err: %d"
% (n, cmd.exit_code))
result = api.types.status.FAILURE
continue
# Copy tech support tar out
# TAR files are created at: pensando/iota/entities/node1_host/<test_case>
resp = api.CopyFromHost(n, ["%s-tech-support.tar.gz" % (n)], tc_dir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy techsupport file from node: %s" %
n)
result = api.types.status.FAILURE
continue
if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN:
return api.types.status.CRITICAL
return result
def runSinlgeTest(test_name, tc):
if api.IsDryrun():
return api.types.status.SUCCESS
srv = tc.srv
cli = tc.cli
req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True)
req2 = api.Trigger_CreateExecuteCommandsRequest(serial=False)
valid_csum = getattr(tc.iterators, 'csum', 'valid')
proto = getattr(tc.iterators, "proto", 'icmp')
ipproto = getattr(tc.iterators, "ipproto", 'v4')
tx_check_sum = getattr(tc.iterators, "tx_check_sum", 'on')
rx_check_sum = getattr(tc.iterators, "rx_check_sum", 'on')
if ipproto == 'v6':
srv_ip_address = srv.ipv6_address
cli_ip_address = cli.ipv6_address
tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\
(srv.interface, srv.ipv6_address, cli.interface, cli.ipv6_address)
else:
srv_ip_address = srv.ip_address
cli_ip_address = cli.ip_address
tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\
(srv.interface, srv.ip_address, cli.interface, cli.ip_address)
api.Logger.info("Starting Checksum [%s] %s" % (test_name, tc.cmd_descr))
api.Logger.info("proto: %s/%s " % (proto, ipproto))
tc.srv_bad_csum = ionic_stats.getNetstatBadCsum(srv, proto)
tc.cli_bad_csum = ionic_stats.getNetstatBadCsum(cli, proto)
# Start TCPdump on server and client.
tc.srv_pcap_file = getPcapFileName("srv_", proto, ipproto, valid_csum, tx_check_sum, rx_check_sum, srv.interface)
status, srv_resp = startTcpDump(srv.node_name, srv.interface, tc.srv_pcap_file, cli_ip_address)
tc.cli_pcap_file = getPcapFileName("cli_", proto, ipproto, valid_csum, tx_check_sum, rx_check_sum, cli.interface)
status, cli_resp = startTcpDump(cli.node_name, cli.interface, tc.cli_pcap_file, srv_ip_address)
# Run the client command.
if test_name == "nping":
client_cmd = getNpingCmd(srv_ip_address, proto, valid_csum, ipproto)
else:
client_cmd = getHping3Cmd(srv_ip_address, proto, valid_csum, ipproto)
api.Trigger_AddCommand(
req2, cli.node_name, cli.workload_name, client_cmd)
tc.resp = api.Trigger(req2)
if tc.resp is None:
api.Logger.error("Failed to run %s on client: %s\n" %(client_cmd, cli.node_name))
return api.types.status.FAILURE
tc.check_rx = False
for cmd in tc.resp.commands:
#
# Bad checksum packets:
# nping command can fail for corrupted packets.
if cmd.exit_code != 0 and valid_csum == "valid":
api.Logger.error("Failed to start %s for valid packet\n"
%(client_cmd))
api.PrintCommandResults(cmd)
return api.types.status.FAILURE
elif cmd.exit_code != 0 and valid_csum != "valid":
api.Logger.info("Failed as expected %s for invalid csum"
%(client_cmd))
return api.types.status.SUCCESS
elif cmd.exit_code != 0:
api.Logger.error("Failed to run %s, Tx csum: %s csum: %s\n"
% (client_cmd, tx_check_sum, valid_csum))
api.PrintCommandResults(cmd)
return api.types.status.FAILURE
# Data was transferred, check for Rx checksum counters.
tc.check_rx = True;
vlan = getattr(tc.iterators, 'vlantag', 'off')
vxlan = getattr(tc.iterators, 'vxlan', 'off')
api.Logger.info("Tx csum: %s Rx csum: %s VLAN: %s VXLAN: %s Proto: %s/%s Checksum: %s"
% (tx_check_sum, rx_check_sum, vlan, vxlan, proto, ipproto, valid_csum))
api.Logger.info("Success running csum %s\n" %(client_cmd))
dir_path = os.path.dirname(os.path.realpath(__file__))
resp = api.CopyFromHost(cli.node_name, [tc.cli_pcap_file], dir_path)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy client file: %s\n"
% (tc.cli_pcap_file))
return api.types.status.FAILURE
resp = api.CopyFromHost(srv.node_name, [tc.srv_pcap_file], dir_path)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy server file: %s\n"
% (tc.srv_pcap_file))
return api.types.status.FAILURE
status = verifySingle(tc)
return status