def buildNodesFromTestbedFile(testbed): Logger.debug("building nodes from testbed file {0}".format(testbed)) nodes = [] try: with open(testbed, "r") as topoFile: data = json.load(topoFile) if 'Vars' in data['Provision'] and getattr( data['Provision']['Vars'], "EsxUsername", None): username = data['Provision']['Vars']['EsxUsername'] password = data['Provision']['Vars']['EsxPassword'] else: username = data['Provision']['Username'] password = data['Provision']['Password'] for inst in data["Instances"]: if inst.get( "Type", "" ) == "bm" and "NodeMgmtIP" in inst and "Name" in inst: node = CollectLogNode(inst["Name"], inst["NodeMgmtIP"], username, password) node.cimcIp = inst.get("NodeCimcIP", None) node.cimcUsername = inst.get("NodeCimcUsername", None) node.cimcPassword = inst.get("NodeCimcPassword", None) nodes.append(node) except: msg = "failed to build nodes from testbed file. error was:" msg += traceback.format_exc() print(msg) Logger.debug(msg) return nodes
def __load_bundle(self): pkg_base = self.__parent.GetPackages()[0].replace(".", "/") fullpath = "%s/%s/testbundles/%s" % (api.GetTopDir(), pkg_base, self.__bunfile) Logger.debug("Importing Testbundle %s" % fullpath) self.__spec = self.__read_spec(fullpath) return
def CleanupTestbed(self): msg = self.__prepare_TestBedMsg(self.prev_ts) resp = api.CleanupTestbed(msg) if resp is None: Logger.error("Failed to cleanup testbed: ") return types.status.FAILURE return types.status.SUCCESS
def __resolve_testcases(self): for tc_spec in self.__spec.testcases: if getattr(tc_spec, 'disable', False): Logger.info("Skipping disabled test case %s" % tc_spec.name) continue tc_spec.packages = self.__parent.GetPackages() if getattr(tc_spec, 'verifs', None): tc_spec.verifs += self.__parent.GetVerifs() else: tc_spec.verifs = self.__parent.GetVerifs() if getattr(tc_spec, 'setups', None): tc_spec.setups += self.__parent.GetSetups() else: tc_spec.setups = self.__parent.GetSetups() if getattr(tc_spec, 'debugs', None): tc_spec.debugs += self.__parent.GetDebugs() else: tc_spec.debugs = self.__parent.GetDebugs() tc_spec.triggers = self.__parent.GetTriggers() if getattr(tc_spec, 'teardowns', None): tc_spec.teardowns += self.__parent.GetTeardowns() else: tc_spec.teardowns = self.__parent.GetTeardowns() if getattr(tc_spec, 'args', None): tc_spec.args.ImportProperties(self.__parent.GetCommonArgs()) else: tc_spec.args = self.__parent.GetCommonArgs() tc = testcase.Testcase(tc_spec, self) self.__testcases.append(tc) return types.status.SUCCESS
def __resolve_teardown(self): teardown_spec = getattr(self.__spec, 'teardown', []) if teardown_spec is None: return types.status.SUCCESS for s in self.__spec.teardown: Logger.debug("Resolving teardown module: %s" % s.step) s.step = loader.Import(s.step, self.__spec.packages) return types.status.SUCCESS
def __parse_setup_topology(self): topospec = getattr(self.__spec.setup, 'topology', None) if not topospec: Logger.error("Error: No topology specified in the testsuite.") assert(0) self.__topology = topology.Topology(topospec, self.ProvisionInfo()) store.SetTopology(self.__topology) return types.status.SUCCESS
def __discover_testsuites(): suites = [] expr = GlobalOptions.topdir + '/iota/test/%s/**/*.testsuite' %\ (GlobalOptions.pipeline) for filename in glob.iglob(expr, recursive=True): Logger.debug("Reading testsuite: %s" % filename) data = parser.YmlParse(filename) suites.append(data) return suites
def Main(): api.Init() # Parse all the testsuites ts_specs = __discover_testsuites() testsuites = [] result = types.status.SUCCESS skipped = 0 for ts_spec in ts_specs: ts = testsuite.TestSuite(ts_spec) testsuites.append(ts) if ts.IsSkipped(): skipped += 1 continue try: ret = ts.Main() except OfflineTestbedException: result = types.status.OFFLINE_TESTBED break #Help in skip setup option finally: if ts.SetupComplete(): SaveSuiteInfo(ts_spec) if ret != types.status.SUCCESS: result = ret if GlobalOptions.no_keep_going: break if result == types.status.OFFLINE_TESTBED: break for ts in testsuites: ts.PrintReport() for ts in testsuites: ts.PrintBundleSummary() print("\n\nOverall Run Summary") print(types.HEADER_SHORT_SUMMARY) print(types.FORMAT_ALL_TESTSUITE_SUMMARY %\ ("Testsuite", "Pass", "Fail", "Ignore", "Dis/Err", "Total", "Target", "%Done", "Result", "Duration")) print(types.HEADER_SHORT_SUMMARY) for ts in testsuites: ts.PrintSummary() print(types.HEADER_SHORT_SUMMARY) if GlobalOptions.coverage_file and GlobalOptions.coverage_dir: api.GetCoverageFiles(GlobalOptions.coverage_file, GlobalOptions.coverage_dir) if skipped == len(ts_specs): #result = types.status.FAILURE #print(40*"#") Logger.warn( "ALL TESTS MARKED AS SKIPPED. NO TESTS RUN. SEE LOG FOR DETAILS.") #print(40*"#") return result
def CollectCores(self): try: destCoreDir = "corefiles/{0}".format(re.sub('[\W]+','_',self.Name())) print("Searching for corefiles.....") core_collector.CollectCores(GlobalOptions.testbed_json, destCoreDir, store.GetTestbed().GetProvisionUsername(), store.GetTestbed().GetProvisionPassword(), Logger) except: Logger.debug("failed to collect cores. error was {0}".format(traceback.format_exc()))
def __setup_config(self): for s in self.__spec.setup.config: # Reset the running directory before every step Logger.info("Starting Config Step: ", s.step) api.ChangeDirectory(None) status = loader.RunCallback(s.step, 'Main', True, getattr(s, "args", None)) if status != types.status.SUCCESS: Logger.error("ERROR: Failed to run config step", s.step) return status return types.status.SUCCESS
def AllocateInstance(self, type, tag=None): for instance in self.__instpool: if instance.Type == type: if tag != None and tag != getattr(instance, "Tag", None): continue self.__instpool.remove(instance) return instance else: Logger.error("No Nodes available in Testbed of type : %s" % type) sys.exit(1)
def __cleanup_testbed_script(self): logfile = "%s/%s_cleanup.log" % (GlobalOptions.logdir, self.curr_ts.Name()) Logger.info("Cleaning up Testbed, Logfile = %s" % logfile) cmd = "timeout 60 ./scripts/cleanup_testbed.py --testbed %s" % GlobalOptions.testbed_json if GlobalOptions.rerun: cmd = cmd + " --rerun" if os.system("%s > %s 2>&1" % (cmd, logfile)) != 0: Logger.info("Cleanup testbed failed.") return types.status.FAILURE return types.status.SUCCESS
def Start(self): Logger.info("Starting IOTA Server: %s (Logfile = %s)" % (self.__cmd, self.__logfile)) log = open(self.__logfile, "w") self.__p = Popen(self.__cmd, stdout=log, stderr=log, shell=True, preexec_fn=preexec_function) return
def InitLogger(): logdir = '%s/iota/logs' % (glopts.GlobalOptions.topdir) os.system("mkdir -p %s" % logdir) if glopts.GlobalOptions.debug: Logger.SetLoggingLevel(types.loglevel.DEBUG) elif glopts.GlobalOptions.verbose: Logger.SetLoggingLevel(types.loglevel.VERBOSE) else: Logger.SetLoggingLevel(types.loglevel.INFO) return
def __resolve_setup_config(self): cfgspec = getattr(self.__spec.setup, 'config', None) if not cfgspec: return types.status.SUCCESS for s in self.__spec.setup.config: Logger.debug("Resolving config step: %s" % s.step) s.name = s.step args = getattr(s, "args", None) s.step = loader.Import(s.step, self.__spec.packages) s.args = getattr(s, "args", None) return types.status.SUCCESS
def __exit_cleanup(): global gl_srv_process if not engine.SkipSetupValid(): Logger.debug("ATEXIT: Stopping IOTA Server as setup was not complete") gl_srv_process.Stop() if glopts.GlobalOptions.dryrun or glopts.GlobalOptions.skip_logs: return packageCoresDirectory() Logger.info("Saving logs to iota_sanity_logs.tar.gz") os.system("%s/iota/scripts/savelogs.sh %s" % (topdir, topdir)) return
def GetPipelineFromProvisionInfo(self, nodeName, nicName): try: prov_spec = parser.YmlParse(self.ProvisionInfo()) nodes = getattr(prov_spec, 'nodes', []) for node in nodes: if node.node.name == nodeName: for nic in node.node.nics: if nic.nic.name == nicName: return nic.nic.pipeline except: Logger.debug("failed to find pipeline for {0}:{1}".format(nodeName, nicName))
def writeTestResults(self): filename = "testsuite_{0}_results.json".format(self.Name()) try: tsr = TestSuiteResults() for tbun in self.__testbundles: if tbun.getTestBundleResults(): tsr.Testcases.extend(tbun.getTestBundleResults()) with open(filename,'w') as outfile: json.dump(tsr, outfile, default=lambda x: x.__dict__, sort_keys=True) except: Logger.debug("failed to save test results to file {0}. error was: {1}".format(filename,traceback.format_exc()))
def __invoke_teardown(self): teardown_spec = getattr(self.__spec, 'teardown', []) if teardown_spec is None: return types.status.SUCCESS for s in teardown_spec: Logger.info("Starting Teardown Step: ", s.step) status = loader.RunCallback(s.step, 'Main', True, getattr(s, "args", None)) if status != types.status.SUCCESS: Logger.error("ERROR: Failed to run teardown step", s.step) return status return types.status.SUCCESS
def TriggerBackgroundTasks(self, tasks, iter_data): result = types.status.SUCCESS for task in tasks: bt = self.__background_tasks[task_name] assert (bt) Logger.debug("Triggering BackgroundTask %s - manual trigger" % task_name) bt_trigger_result = bt.StartTask(self.__tc, iter_data) if bt_trigger_result != types.status.SUCCESS: result = bt_trigger_result return result
def wrapped(*args, **kwargs): signal.signal(signal.SIGALRM, alarmHandler) try: seconds = args[0].GetTimeout() except: seconds = defaultTcTimeout Logger.info("testcase timeout: {0}".format(seconds)) signal.alarm(seconds) try: return func(*args, **kwargs) finally: signal.alarm(0)
def __resolve(self): Logger.debug("Resolving testcase teardown module: %s" % self.__spec.step) self.__mod = loader.Import(self.__spec.step, self.__spec.packages) if hasattr(self.__spec, 'args') and hasattr(self.__spec.args, 'trigger'): self.__trigger = self.__spec.args.trigger if hasattr(self.__spec, 'args') and hasattr(self.__spec.args, 'background'): self.__background = self.__spec.args.background return
def StopBackgroundTasks(self, tasks): result = types.status.SUCCESS for task in tasks: bt = self.__background_tasks[task_name] assert (bt) Logger.debug("Triggering BackgroundTask %s - manual trigger" % task_name) bt_trigger_result = bt.CollectTask() if bt_trigger_result != types.status.SUCCESS: result = bt_trigger_result return result
def GetImageManifestFile(self): if GlobalOptions.compat_test: Logger.debug("Compat-testing for driver: %s and firmware: %s" % (GlobalOptions.driver_version, GlobalOptions.fw_version)) # assert(GlobalOptions.driver_version != GlobalOptions.fw_version) return self.__build_new_image_manifest() else: Logger.debug("Using testsuite spec for image-manifest") if hasattr(self.__spec, 'image_manifest'): path = getattr(self.__spec.image_manifest, 'file', 'images/latest.json') else: path = 'images/latest.json' return os.path.join(GlobalOptions.topdir, path)
def __start_server(): Logger.debug("setting default SIGINT handler") signal.signal(signal.SIGINT, signal.default_int_handler) global gl_srv_process srv_binary = "VENICE_DEV=1 nohup %s/iota/bin/server/iota_server" % topdir srv_logfile = "%s/server.log" % glopts.GlobalOptions.logdir srv_args = "--port %d" % glopts.GlobalOptions.svcport if glopts.GlobalOptions.dryrun: srv_args += " --stubmode" gl_srv_process = procs.IotaProcess("%s %s" % (srv_binary, srv_args), srv_logfile) gl_srv_process.Start() return
def __execute(self, iter_data): if self.__background: Logger.debug("Running common teardown module in background: %s" % self.__spec.step) bt_inst = TestcaseBackgroundTrigger('Main', self.__trigger, self.__run, self.__sleep_interval, self.__terminate) return bt_inst.StartTask(self.__mod, iter_data) else: Logger.debug("Running common teardown module in foreground: %s" % self.__spec.step) return loader.RunCallback(self.__mod, 'Main', True, iter_data)
def __init__(self, msg=None, running=True): self.running = running if msg: self.workload_name = msg.workload_name self.workload_type = msg.workload_type self.workload_image = msg.workload_image self.node_name = msg.node_name self.encap_vlan = msg.interfaces[0].encap_vlan self.ip_prefix = msg.interfaces[0].ip_prefix self.ip_address = msg.interfaces[0].ip_prefix.split('/')[0] self.sec_ip_prefixes = [] self.sec_ip_addresses = [] for secip in msg.interfaces[0].sec_ip_prefix: self.sec_ip_prefixes.append(secip) self.sec_ip_addresses.append(secip.split('/')[0]) self.ipv6_prefix = msg.interfaces[0].ipv6_prefix self.ipv6_address = msg.interfaces[0].ipv6_prefix.split('/')[0] self.sec_ipv6_prefixes = [] self.sec_ipv6_addresses = [] for secip in msg.interfaces[0].sec_ipv6_prefix: self.sec_ipv6_prefixes.append(secip) self.sec_ipv6_addresses.append(secip.split('/')[0]) self.mac_address = msg.interfaces[0].mac_address self.interface = msg.interfaces[0].interface self.interfaces = msg.interfaces self.parent_interface = msg.interfaces[0].parent_interface self.interface_type = msg.interfaces[0].interface_type self.pinned_port = msg.interfaces[0].pinned_port self.uplink_vlan = msg.interfaces[0].uplink_vlan self.device_name = msg.interfaces[0].device_name self.cpus = msg.cpus self.memory = msg.memory self.mgmt_ip = msg.mgmt_ip self.network_name = msg.interfaces[0].network_name #Logger.info("Workload %s Mgmt IP %s" % (self.workload_name, self.mgmt_ip)) self.exposed_tcp_ports = [] self.exposed_udp_ports = [] for exposed_port in msg.exposed_ports: if exposed_port.Proto == "tcp": self.exposed_tcp_ports.append(exposed_port.Port) if exposed_port.Proto == "udp": self.exposed_udp_ports.append(exposed_port.Port) if len(self.exposed_tcp_ports) != 0: Logger.info("Worklaod %s exposed tcp ports %s" % (self.workload_name, self.exposed_tcp_ports)) if len(self.exposed_udp_ports) != 0: Logger.info("Workload %s exposed udp ports %s" % (self.workload_name, self.exposed_udp_ports)) return
def ExitHandler(self): if GlobalOptions.dryrun: return status = self.__teardown() Logger.info(f"Testsuite {self.Name()} teardown STATUS = {status}") api.UnsetBreakoutInterfaces() logcollector.CollectLogs() logcollector.CollectTechSupport(self.Name()) self.CollectCores() self.writeTestResults() if store.GetTestbed().IsUsingProdVCenter(): store.GetTestbed().CleanupTestbed() store.GetTopology().CleanupTopology() return
def SkipSetupValid(): if len(GlobalOptions.testsuites) != 1: return False try: with open(SuiteRunFile, 'r') as json_file: data = json.load(json_file) return data.get("name", None) == GlobalOptions.testsuites[0] and data.get( "setupDone", False) except: pass Logger.info("Skip setup not valid...") return False
def checkPci(self): result = types.status.SUCCESS if GlobalOptions.dryrun: return result for node in self.GetTopology().GetNodes(): msg = "calling verify_pci.verify_error_lspci() for node {0}".format(node.Name()) Logger.debug(msg) result = verify_pci.verify_errors_lspci(node.Name(), node.GetOs()) if result != types.status.SUCCESS: msg = "PCIe Failure detected on node {0} with OS {1}".format(node.Name(), node.GetOs()) print(msg) Logger.error(msg) return result # raise OfflineTestbedException return result