def Trigger(tc): # skip some iterator cases if skip_curr_test(tc): return api.types.status.SUCCESS req = api.Trigger_CreateExecuteCommandsRequest() trig_cmd1 = "cd %s; tar -czvf %s output" % (DTS_OUTPUT_PATH, DTS_LOG_TAR_FILE) api.Trigger_AddHostCommand(req, tc.tester_node.Name(), trig_cmd1, timeout=60) trig_resp = api.Trigger(req) tc.resp.append(trig_resp) tc_dir = tc.GetLogsDir() api.Logger.info("Dest dir is %s: Src is %s" % (tc_dir, DTS_OUTPUT_PATH + DTS_LOG_TAR_FILE)) resp = api.CopyFromHost(tc.tester_node_name, [DTS_OUTPUT_PATH + DTS_LOG_TAR_FILE], tc_dir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy dts log file from node: %s" % tc.tester_node_name) result = api.types.status.FAILURE return api.types.status.SUCCESS
def stop_single_pcap_capture(tc): if not getattr(tc, "pcap_trigger", None): return api.types.status.SUCCESS api.Trigger_TerminateAllCommands(tc.pcap_trigger) nodes = api.GetWorkloadNodeHostnames() tc_dir = tc.GetLogsDir() resp = api.CopyFromHost(n, [tc.pcap_filename], tc_dir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy from to Node: %s" % n) return api.types.status.FAILURE return api.types.status.SUCCESS
def CollectTechSupport(tsName): try: #global __CURREN_TECHSUPPORT_CNT #__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1 if GlobalOptions.pipeline in ["apulu"]: return types.status.SUCCESS Logger.info("Collecting techsupport for testsuite {0}".format(tsName)) tsName = re.sub('\W', '_', tsName) logDir = GlobalOptions.logdir if not logDir.endswith('/'): logDir += '/' logDir += 'techsupport/' if not os.path.exists(logDir): os.mkdir(logDir) nodes = api.GetNaplesHostnames() req = api.Trigger_CreateExecuteCommandsRequest() for n in nodes: Logger.info("Techsupport for node: %s" % n) common.AddPenctlCommand( req, n, "system tech-support -b %s-tech-support" % (n)) resp = api.Trigger(req) result = types.status.SUCCESS for n, cmd in zip(nodes, resp.commands): #api.PrintCommandResults(cmd) if cmd.exit_code != 0: Logger.error( "Failed to execute penctl system tech-support on node: %s. err: %d" % (n, cmd.exit_code)) result = types.status.FAILURE continue # Copy tech support tar out # TAR files are created at: pensando/iota/entities/node1_host/<test_case> ntsn = "%s-tech-support.tar.gz" % (n) resp = api.CopyFromHost(n, [ntsn], logDir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: Logger.error( "Failed to copy techsupport file %s from node: %s" % (ntsn, n)) result = types.status.FAILURE continue os.rename(logDir + ntsn, logDir + tsName + '_' + ntsn) #if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN: # return types.status.CRITICAL return result except AttributeError: Logger.debug('failed to collect tech support. node list not setup yet') except: Logger.debug('failed to collect tech support. error was: {0}'.format( traceback.format_exc())) return types.status.CRITICAL
def Verify(tc): for cmd in tc.install_cmd_resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd): api.Logger.error("Installation failed on Node %s\n" % (cmd.node_name)) return api.types.status.FAILURE for cmd in tc.deploy_cmd_resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd): api.Logger.error("Deploy failed on Node %s\n" % (cmd.node_name)) return api.types.status.FAILURE for cmd in tc.validate_cmd_resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0 and not api.Trigger_IsBackgroundCommand(cmd): api.Logger.error("Validation procedure failed on Node %s\n" % (cmd.node_name)) return api.types.status.FAILURE dir_path = os.path.dirname(os.path.realpath(__file__)) api.Logger.info("Dir Path is: %s\n" % (dir_path)) for n in tc.nodes: api.Logger.info("Getting data for: %s\n" % (n)) resp = api.CopyFromHost(n, ["pencap/linux/techsupport/techsupport.json"], dir_path) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy client file: %s\n" % (n)) return api.types.status.FAILURE jdata = {} try: with open(dir_path + "/techsupport.json") as f: jdata = json.load(f) except: api.Logger.error("Unable to open local file from node: %s\n" % (n)) return api.types.status.FAILURE for data in jdata: array_of_dumps = jdata[data] for dump in array_of_dumps: dump_menu_title = "" for item in dump: if (isinstance(dump[item], str) == True): dump_menu_title += dump[item] + "_" api.Logger.info("Dumped data: %s\n" % (dump_menu_title)) return api.types.status.SUCCESS
def stop_pcap_capture(tc): api.Trigger_TerminateAllCommands(tc.pcap_trigger) nodes = api.GetWorkloadNodeHostnames() tc_dir = tc.GetLogsDir() for n, host_intfs in tc.host_intfs.items(): if len(host_intfs) == 0: api.Logger.error("No host interfaces for node :%s" % n) return api.types.status.FAILURE for intfObj in host_intfs: intf = intfObj.Name() resp = api.CopyFromHost(n, [pcap_file_name(intf)], tc_dir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy from to Node: %s" % n) return api.types.status.FAILURE return api.types.status.SUCCESS
def Main(tc): if tc.GetStatus() == api.types.status.SUCCESS: return api.types.status.SUCCESS global __CURREN_TECHSUPPORT_CNT __CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1 api.Logger.info("TC: %s failed. Collecting techsupport." % (tc.Name())) nodes = api.GetNaplesHostnames() req = api.Trigger_CreateExecuteCommandsRequest() tc_dir = tc.GetLogsDir() for n in nodes: api.Logger.info("Techsupport for node: %s" % n) common.AddPenctlCommand(req, n, "system tech-support -b %s-tech-support" % (n)) tc.resp = api.Trigger(req) result = api.types.status.SUCCESS for n, cmd in zip(nodes, tc.resp.commands): api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error( "Failed to execute penctl system tech-support on node: %s. err: %d" % (n, cmd.exit_code)) result = api.types.status.FAILURE continue # Copy tech support tar out # TAR files are created at: pensando/iota/entities/node1_host/<test_case> resp = api.CopyFromHost(n, ["%s-tech-support.tar.gz" % (n)], tc_dir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy techsupport file from node: %s" % n) result = api.types.status.FAILURE continue if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN: return api.types.status.CRITICAL return result
def runSinlgeTest(test_name, tc): if api.IsDryrun(): return api.types.status.SUCCESS srv = tc.srv cli = tc.cli req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True) req2 = api.Trigger_CreateExecuteCommandsRequest(serial=False) valid_csum = getattr(tc.iterators, 'csum', 'valid') proto = getattr(tc.iterators, "proto", 'icmp') ipproto = getattr(tc.iterators, "ipproto", 'v4') tx_check_sum = getattr(tc.iterators, "tx_check_sum", 'on') rx_check_sum = getattr(tc.iterators, "rx_check_sum", 'on') if ipproto == 'v6': srv_ip_address = srv.ipv6_address cli_ip_address = cli.ipv6_address tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\ (srv.interface, srv.ipv6_address, cli.interface, cli.ipv6_address) else: srv_ip_address = srv.ip_address cli_ip_address = cli.ip_address tc.cmd_descr = " Server: %s(%s) <--> Client: %s(%s)" %\ (srv.interface, srv.ip_address, cli.interface, cli.ip_address) api.Logger.info("Starting Checksum [%s] %s" % (test_name, tc.cmd_descr)) api.Logger.info("proto: %s/%s " % (proto, ipproto)) tc.srv_bad_csum = ionic_stats.getNetstatBadCsum(srv, proto) tc.cli_bad_csum = ionic_stats.getNetstatBadCsum(cli, proto) # Start TCPdump on server and client. tc.srv_pcap_file = getPcapFileName("srv_", proto, ipproto, valid_csum, tx_check_sum, rx_check_sum, srv.interface) status, srv_resp = startTcpDump(srv.node_name, srv.interface, tc.srv_pcap_file, cli_ip_address) tc.cli_pcap_file = getPcapFileName("cli_", proto, ipproto, valid_csum, tx_check_sum, rx_check_sum, cli.interface) status, cli_resp = startTcpDump(cli.node_name, cli.interface, tc.cli_pcap_file, srv_ip_address) # Run the client command. if test_name == "nping": client_cmd = getNpingCmd(srv_ip_address, proto, valid_csum, ipproto) else: client_cmd = getHping3Cmd(srv_ip_address, proto, valid_csum, ipproto) api.Trigger_AddCommand( req2, cli.node_name, cli.workload_name, client_cmd) tc.resp = api.Trigger(req2) if tc.resp is None: api.Logger.error("Failed to run %s on client: %s\n" %(client_cmd, cli.node_name)) return api.types.status.FAILURE tc.check_rx = False for cmd in tc.resp.commands: # # Bad checksum packets: # nping command can fail for corrupted packets. if cmd.exit_code != 0 and valid_csum == "valid": api.Logger.error("Failed to start %s for valid packet\n" %(client_cmd)) api.PrintCommandResults(cmd) return api.types.status.FAILURE elif cmd.exit_code != 0 and valid_csum != "valid": api.Logger.info("Failed as expected %s for invalid csum" %(client_cmd)) return api.types.status.SUCCESS elif cmd.exit_code != 0: api.Logger.error("Failed to run %s, Tx csum: %s csum: %s\n" % (client_cmd, tx_check_sum, valid_csum)) api.PrintCommandResults(cmd) return api.types.status.FAILURE # Data was transferred, check for Rx checksum counters. tc.check_rx = True; vlan = getattr(tc.iterators, 'vlantag', 'off') vxlan = getattr(tc.iterators, 'vxlan', 'off') api.Logger.info("Tx csum: %s Rx csum: %s VLAN: %s VXLAN: %s Proto: %s/%s Checksum: %s" % (tx_check_sum, rx_check_sum, vlan, vxlan, proto, ipproto, valid_csum)) api.Logger.info("Success running csum %s\n" %(client_cmd)) dir_path = os.path.dirname(os.path.realpath(__file__)) resp = api.CopyFromHost(cli.node_name, [tc.cli_pcap_file], dir_path) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy client file: %s\n" % (tc.cli_pcap_file)) return api.types.status.FAILURE resp = api.CopyFromHost(srv.node_name, [tc.srv_pcap_file], dir_path) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy server file: %s\n" % (tc.srv_pcap_file)) return api.types.status.FAILURE status = verifySingle(tc) return status