def buildNodesFromTestbedFile(testbed):
Logger.debug("building nodes from testbed file {0}".format(testbed))
nodes = []
try:
with open(testbed, "r") as topoFile:
data = json.load(topoFile)
if 'Vars' in data['Provision'] and getattr(
data['Provision']['Vars'], "EsxUsername", None):
username = data['Provision']['Vars']['EsxUsername']
password = data['Provision']['Vars']['EsxPassword']
else:
username = data['Provision']['Username']
password = data['Provision']['Password']
for inst in data["Instances"]:
if inst.get(
"Type", ""
) == "bm" and "NodeMgmtIP" in inst and "Name" in inst:
node = CollectLogNode(inst["Name"], inst["NodeMgmtIP"],
username, password)
node.cimcIp = inst.get("NodeCimcIP", None)
node.cimcUsername = inst.get("NodeCimcUsername", None)
node.cimcPassword = inst.get("NodeCimcPassword", None)
nodes.append(node)
except:
msg = "failed to build nodes from testbed file. error was:"
msg += traceback.format_exc()
print(msg)
Logger.debug(msg)
return nodes
def __load_bundle(self):
pkg_base = self.__parent.GetPackages()[0].replace(".", "/")
fullpath = "%s/%s/testbundles/%s" % (api.GetTopDir(), pkg_base,
self.__bunfile)
Logger.debug("Importing Testbundle %s" % fullpath)
self.__spec = self.__read_spec(fullpath)
return
def CleanupTestbed(self):
msg = self.__prepare_TestBedMsg(self.prev_ts)
resp = api.CleanupTestbed(msg)
if resp is None:
Logger.error("Failed to cleanup testbed: ")
return types.status.FAILURE
return types.status.SUCCESS
def __resolve_testcases(self):
for tc_spec in self.__spec.testcases:
if getattr(tc_spec, 'disable', False):
Logger.info("Skipping disabled test case %s" % tc_spec.name)
continue
tc_spec.packages = self.__parent.GetPackages()
if getattr(tc_spec, 'verifs', None):
tc_spec.verifs += self.__parent.GetVerifs()
else:
tc_spec.verifs = self.__parent.GetVerifs()
if getattr(tc_spec, 'setups', None):
tc_spec.setups += self.__parent.GetSetups()
else:
tc_spec.setups = self.__parent.GetSetups()
if getattr(tc_spec, 'debugs', None):
tc_spec.debugs += self.__parent.GetDebugs()
else:
tc_spec.debugs = self.__parent.GetDebugs()
tc_spec.triggers = self.__parent.GetTriggers()
if getattr(tc_spec, 'teardowns', None):
tc_spec.teardowns += self.__parent.GetTeardowns()
else:
tc_spec.teardowns = self.__parent.GetTeardowns()
if getattr(tc_spec, 'args', None):
tc_spec.args.ImportProperties(self.__parent.GetCommonArgs())
else:
tc_spec.args = self.__parent.GetCommonArgs()
tc = testcase.Testcase(tc_spec, self)
self.__testcases.append(tc)
return types.status.SUCCESS
def __resolve_teardown(self):
teardown_spec = getattr(self.__spec, 'teardown', [])
if teardown_spec is None:
return types.status.SUCCESS
for s in self.__spec.teardown:
Logger.debug("Resolving teardown module: %s" % s.step)
s.step = loader.Import(s.step, self.__spec.packages)
return types.status.SUCCESS
def __parse_setup_topology(self):
topospec = getattr(self.__spec.setup, 'topology', None)
if not topospec:
Logger.error("Error: No topology specified in the testsuite.")
assert(0)
self.__topology = topology.Topology(topospec, self.ProvisionInfo())
store.SetTopology(self.__topology)
return types.status.SUCCESS
def __discover_testsuites():
suites = []
expr = GlobalOptions.topdir + '/iota/test/%s/**/*.testsuite' %\
(GlobalOptions.pipeline)
for filename in glob.iglob(expr, recursive=True):
Logger.debug("Reading testsuite: %s" % filename)
data = parser.YmlParse(filename)
suites.append(data)
return suites
def Main():
api.Init()
# Parse all the testsuites
ts_specs = __discover_testsuites()
testsuites = []
result = types.status.SUCCESS
skipped = 0
for ts_spec in ts_specs:
ts = testsuite.TestSuite(ts_spec)
testsuites.append(ts)
if ts.IsSkipped():
skipped += 1
continue
try:
ret = ts.Main()
except OfflineTestbedException:
result = types.status.OFFLINE_TESTBED
break
#Help in skip setup option
finally:
if ts.SetupComplete():
SaveSuiteInfo(ts_spec)
if ret != types.status.SUCCESS:
result = ret
if GlobalOptions.no_keep_going:
break
if result == types.status.OFFLINE_TESTBED:
break
for ts in testsuites:
ts.PrintReport()
for ts in testsuites:
ts.PrintBundleSummary()
print("\n\nOverall Run Summary")
print(types.HEADER_SHORT_SUMMARY)
print(types.FORMAT_ALL_TESTSUITE_SUMMARY %\
("Testsuite", "Pass", "Fail", "Ignore", "Dis/Err", "Total", "Target", "%Done", "Result", "Duration"))
print(types.HEADER_SHORT_SUMMARY)
for ts in testsuites:
ts.PrintSummary()
print(types.HEADER_SHORT_SUMMARY)
if GlobalOptions.coverage_file and GlobalOptions.coverage_dir:
api.GetCoverageFiles(GlobalOptions.coverage_file,
GlobalOptions.coverage_dir)
if skipped == len(ts_specs):
#result = types.status.FAILURE
#print(40*"#")
Logger.warn(
"ALL TESTS MARKED AS SKIPPED. NO TESTS RUN. SEE LOG FOR DETAILS.")
#print(40*"#")
return result
def CollectCores(self):
try:
destCoreDir = "corefiles/{0}".format(re.sub('[\W]+','_',self.Name()))
print("Searching for corefiles.....")
core_collector.CollectCores(GlobalOptions.testbed_json, destCoreDir,
store.GetTestbed().GetProvisionUsername(),
store.GetTestbed().GetProvisionPassword(), Logger)
except:
Logger.debug("failed to collect cores. error was {0}".format(traceback.format_exc()))
def __setup_config(self):
for s in self.__spec.setup.config:
# Reset the running directory before every step
Logger.info("Starting Config Step: ", s.step)
api.ChangeDirectory(None)
status = loader.RunCallback(s.step, 'Main', True, getattr(s, "args", None))
if status != types.status.SUCCESS:
Logger.error("ERROR: Failed to run config step", s.step)
return status
return types.status.SUCCESS
def AllocateInstance(self, type, tag=None):
for instance in self.__instpool:
if instance.Type == type:
if tag != None and tag != getattr(instance, "Tag", None):
continue
self.__instpool.remove(instance)
return instance
else:
Logger.error("No Nodes available in Testbed of type : %s" % type)
sys.exit(1)
def __cleanup_testbed_script(self):
logfile = "%s/%s_cleanup.log" % (GlobalOptions.logdir, self.curr_ts.Name())
Logger.info("Cleaning up Testbed, Logfile = %s" % logfile)
cmd = "timeout 60 ./scripts/cleanup_testbed.py --testbed %s" % GlobalOptions.testbed_json
if GlobalOptions.rerun:
cmd = cmd + " --rerun"
if os.system("%s > %s 2>&1" % (cmd, logfile)) != 0:
Logger.info("Cleanup testbed failed.")
return types.status.FAILURE
return types.status.SUCCESS
def Start(self):
Logger.info("Starting IOTA Server: %s (Logfile = %s)" %
(self.__cmd, self.__logfile))
log = open(self.__logfile, "w")
self.__p = Popen(self.__cmd,
stdout=log,
stderr=log,
shell=True,
preexec_fn=preexec_function)
return
def InitLogger():
logdir = '%s/iota/logs' % (glopts.GlobalOptions.topdir)
os.system("mkdir -p %s" % logdir)
if glopts.GlobalOptions.debug:
Logger.SetLoggingLevel(types.loglevel.DEBUG)
elif glopts.GlobalOptions.verbose:
Logger.SetLoggingLevel(types.loglevel.VERBOSE)
else:
Logger.SetLoggingLevel(types.loglevel.INFO)
return
def __resolve_setup_config(self):
cfgspec = getattr(self.__spec.setup, 'config', None)
if not cfgspec:
return types.status.SUCCESS
for s in self.__spec.setup.config:
Logger.debug("Resolving config step: %s" % s.step)
s.name = s.step
args = getattr(s, "args", None)
s.step = loader.Import(s.step, self.__spec.packages)
s.args = getattr(s, "args", None)
return types.status.SUCCESS
def __exit_cleanup():
global gl_srv_process
if not engine.SkipSetupValid():
Logger.debug("ATEXIT: Stopping IOTA Server as setup was not complete")
gl_srv_process.Stop()
if glopts.GlobalOptions.dryrun or glopts.GlobalOptions.skip_logs:
return
packageCoresDirectory()
Logger.info("Saving logs to iota_sanity_logs.tar.gz")
os.system("%s/iota/scripts/savelogs.sh %s" % (topdir, topdir))
return
def GetPipelineFromProvisionInfo(self, nodeName, nicName):
try:
prov_spec = parser.YmlParse(self.ProvisionInfo())
nodes = getattr(prov_spec, 'nodes', [])
for node in nodes:
if node.node.name == nodeName:
for nic in node.node.nics:
if nic.nic.name == nicName:
return nic.nic.pipeline
except:
Logger.debug("failed to find pipeline for {0}:{1}".format(nodeName, nicName))
def writeTestResults(self):
filename = "testsuite_{0}_results.json".format(self.Name())
try:
tsr = TestSuiteResults()
for tbun in self.__testbundles:
if tbun.getTestBundleResults():
tsr.Testcases.extend(tbun.getTestBundleResults())
with open(filename,'w') as outfile:
json.dump(tsr, outfile, default=lambda x: x.__dict__, sort_keys=True)
except:
Logger.debug("failed to save test results to file {0}. error was: {1}".format(filename,traceback.format_exc()))
def __invoke_teardown(self):
teardown_spec = getattr(self.__spec, 'teardown', [])
if teardown_spec is None:
return types.status.SUCCESS
for s in teardown_spec:
Logger.info("Starting Teardown Step: ", s.step)
status = loader.RunCallback(s.step, 'Main', True, getattr(s, "args", None))
if status != types.status.SUCCESS:
Logger.error("ERROR: Failed to run teardown step", s.step)
return status
return types.status.SUCCESS
def TriggerBackgroundTasks(self, tasks, iter_data):
result = types.status.SUCCESS
for task in tasks:
bt = self.__background_tasks[task_name]
assert (bt)
Logger.debug("Triggering BackgroundTask %s - manual trigger" %
task_name)
bt_trigger_result = bt.StartTask(self.__tc, iter_data)
if bt_trigger_result != types.status.SUCCESS:
result = bt_trigger_result
return result
def wrapped(*args, **kwargs):
signal.signal(signal.SIGALRM, alarmHandler)
try:
seconds = args[0].GetTimeout()
except:
seconds = defaultTcTimeout
Logger.info("testcase timeout: {0}".format(seconds))
signal.alarm(seconds)
try:
return func(*args, **kwargs)
finally:
signal.alarm(0)
def __resolve(self):
Logger.debug("Resolving testcase teardown module: %s" %
self.__spec.step)
self.__mod = loader.Import(self.__spec.step, self.__spec.packages)
if hasattr(self.__spec, 'args') and hasattr(self.__spec.args,
'trigger'):
self.__trigger = self.__spec.args.trigger
if hasattr(self.__spec, 'args') and hasattr(self.__spec.args,
'background'):
self.__background = self.__spec.args.background
return
def StopBackgroundTasks(self, tasks):
result = types.status.SUCCESS
for task in tasks:
bt = self.__background_tasks[task_name]
assert (bt)
Logger.debug("Triggering BackgroundTask %s - manual trigger" %
task_name)
bt_trigger_result = bt.CollectTask()
if bt_trigger_result != types.status.SUCCESS:
result = bt_trigger_result
return result
def GetImageManifestFile(self):
if GlobalOptions.compat_test:
Logger.debug("Compat-testing for driver: %s and firmware: %s" % (GlobalOptions.driver_version, GlobalOptions.fw_version))
# assert(GlobalOptions.driver_version != GlobalOptions.fw_version)
return self.__build_new_image_manifest()
else:
Logger.debug("Using testsuite spec for image-manifest")
if hasattr(self.__spec, 'image_manifest'):
path = getattr(self.__spec.image_manifest, 'file', 'images/latest.json')
else:
path = 'images/latest.json'
return os.path.join(GlobalOptions.topdir, path)
def __start_server():
Logger.debug("setting default SIGINT handler")
signal.signal(signal.SIGINT, signal.default_int_handler)
global gl_srv_process
srv_binary = "VENICE_DEV=1 nohup %s/iota/bin/server/iota_server" % topdir
srv_logfile = "%s/server.log" % glopts.GlobalOptions.logdir
srv_args = "--port %d" % glopts.GlobalOptions.svcport
if glopts.GlobalOptions.dryrun:
srv_args += " --stubmode"
gl_srv_process = procs.IotaProcess("%s %s" % (srv_binary, srv_args),
srv_logfile)
gl_srv_process.Start()
return
def __execute(self, iter_data):
if self.__background:
Logger.debug("Running common teardown module in background: %s" %
self.__spec.step)
bt_inst = TestcaseBackgroundTrigger('Main', self.__trigger,
self.__run,
self.__sleep_interval,
self.__terminate)
return bt_inst.StartTask(self.__mod, iter_data)
else:
Logger.debug("Running common teardown module in foreground: %s" %
self.__spec.step)
return loader.RunCallback(self.__mod, 'Main', True, iter_data)
def __init__(self, msg=None, running=True):
self.running = running
if msg:
self.workload_name = msg.workload_name
self.workload_type = msg.workload_type
self.workload_image = msg.workload_image
self.node_name = msg.node_name
self.encap_vlan = msg.interfaces[0].encap_vlan
self.ip_prefix = msg.interfaces[0].ip_prefix
self.ip_address = msg.interfaces[0].ip_prefix.split('/')[0]
self.sec_ip_prefixes = []
self.sec_ip_addresses = []
for secip in msg.interfaces[0].sec_ip_prefix:
self.sec_ip_prefixes.append(secip)
self.sec_ip_addresses.append(secip.split('/')[0])
self.ipv6_prefix = msg.interfaces[0].ipv6_prefix
self.ipv6_address = msg.interfaces[0].ipv6_prefix.split('/')[0]
self.sec_ipv6_prefixes = []
self.sec_ipv6_addresses = []
for secip in msg.interfaces[0].sec_ipv6_prefix:
self.sec_ipv6_prefixes.append(secip)
self.sec_ipv6_addresses.append(secip.split('/')[0])
self.mac_address = msg.interfaces[0].mac_address
self.interface = msg.interfaces[0].interface
self.interfaces = msg.interfaces
self.parent_interface = msg.interfaces[0].parent_interface
self.interface_type = msg.interfaces[0].interface_type
self.pinned_port = msg.interfaces[0].pinned_port
self.uplink_vlan = msg.interfaces[0].uplink_vlan
self.device_name = msg.interfaces[0].device_name
self.cpus = msg.cpus
self.memory = msg.memory
self.mgmt_ip = msg.mgmt_ip
self.network_name = msg.interfaces[0].network_name
#Logger.info("Workload %s Mgmt IP %s" % (self.workload_name, self.mgmt_ip))
self.exposed_tcp_ports = []
self.exposed_udp_ports = []
for exposed_port in msg.exposed_ports:
if exposed_port.Proto == "tcp":
self.exposed_tcp_ports.append(exposed_port.Port)
if exposed_port.Proto == "udp":
self.exposed_udp_ports.append(exposed_port.Port)
if len(self.exposed_tcp_ports) != 0:
Logger.info("Worklaod %s exposed tcp ports %s" %
(self.workload_name, self.exposed_tcp_ports))
if len(self.exposed_udp_ports) != 0:
Logger.info("Workload %s exposed udp ports %s" %
(self.workload_name, self.exposed_udp_ports))
return
def ExitHandler(self):
if GlobalOptions.dryrun:
return
status = self.__teardown()
Logger.info(f"Testsuite {self.Name()} teardown STATUS = {status}")
api.UnsetBreakoutInterfaces()
logcollector.CollectLogs()
logcollector.CollectTechSupport(self.Name())
self.CollectCores()
self.writeTestResults()
if store.GetTestbed().IsUsingProdVCenter():
store.GetTestbed().CleanupTestbed()
store.GetTopology().CleanupTopology()
return
def SkipSetupValid():
if len(GlobalOptions.testsuites) != 1:
return False
try:
with open(SuiteRunFile, 'r') as json_file:
data = json.load(json_file)
return data.get("name",
None) == GlobalOptions.testsuites[0] and data.get(
"setupDone", False)
except:
pass
Logger.info("Skip setup not valid...")
return False
def checkPci(self):
result = types.status.SUCCESS
if GlobalOptions.dryrun:
return result
for node in self.GetTopology().GetNodes():
msg = "calling verify_pci.verify_error_lspci() for node {0}".format(node.Name())
Logger.debug(msg)
result = verify_pci.verify_errors_lspci(node.Name(), node.GetOs())
if result != types.status.SUCCESS:
msg = "PCIe Failure detected on node {0} with OS {1}".format(node.Name(), node.GetOs())
print(msg)
Logger.error(msg)
return result
# raise OfflineTestbedException
return result
