def sanity_check(localhost, duthost, request, fanouthosts, tbinfo):
logger.info("Start pre-test sanity check")
skip_sanity = False
allow_recover = False
recover_method = "adaptive"
check_items = set(copy.deepcopy(
constants.DEFAULT_CHECK_ITEMS)) # Default check items
post_check = False
customized_sanity_check = None
for m in request.node.iter_markers():
logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" %
(m.name, m.args, m.kwargs))
if m.name == "sanity_check":
customized_sanity_check = m
break
if customized_sanity_check:
logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" %
(m.name, str(m.args), str(m.kwargs)))
skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
allow_recover = customized_sanity_check.kwargs.get(
"allow_recover", False)
recover_method = customized_sanity_check.kwargs.get(
"recover_method", "adaptive")
if allow_recover and recover_method not in constants.RECOVER_METHODS:
pytest.warning("Unsupported recover method")
logger.info(
"Fall back to use default recover method 'config_reload'")
recover_method = "config_reload"
check_items = _update_check_items(
check_items, customized_sanity_check.kwargs.get("check_items", []),
constants.SUPPORTED_CHECK_ITEMS)
post_check = customized_sanity_check.kwargs.get("post_check", False)
if request.config.option.skip_sanity:
skip_sanity = True
if request.config.option.allow_recover:
allow_recover = True
items = request.config.getoption("--check_items")
if items:
items_array = str(items).split(',')
check_items = _update_check_items(check_items, items_array,
constants.SUPPORTED_CHECK_ITEMS)
# ignore BGP check for particular topology type
if tbinfo['topo']['type'] == 'ptf' and 'bgp' in check_items:
check_items.remove('bgp')
logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \
(skip_sanity, check_items, allow_recover, recover_method, post_check))
if skip_sanity:
logger.info(
"Skip sanity check according to command line argument or configuration of test script."
)
yield
return
if not check_items:
logger.info(
"No sanity check item is specified, no pre-test sanity check")
yield
logger.info(
"No sanity check item is specified, no post-test sanity check")
return
print_logs(duthost, constants.PRINT_LOGS)
check_results = do_checks(duthost, check_items)
logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(check_results, indent=4))
if any([result["failed"] for result in check_results]):
if not allow_recover:
pytest.fail(
"Pre-test sanity check failed, allow_recover=False {}".format(
check_results))
return
logger.info(
"Pre-test sanity check failed, try to recover, recover_method=%s" %
recover_method)
recover(duthost, localhost, fanouthosts, check_results, recover_method)
logger.info("Run sanity check again after recovery")
new_check_results = do_checks(duthost, check_items)
logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(new_check_results, indent=4))
if any([result["failed"] for result in new_check_results]):
failed_items = json.dumps(
[result for result in new_check_results if result["failed"]],
indent=4)
logger.error("Failed check items:\n{}".format(failed_items))
pytest.fail(
"Pre-test sanity check failed again after recovered by '{}' with failed items:\n{}"
.format(recover_method, failed_items))
return
logger.info("Done pre-test sanity check")
yield
logger.info("Start post-test sanity check")
if not post_check:
logger.info(
"No post-test check is required. Done post-test sanity check")
return
post_check_results = do_checks(duthost, check_items)
logger.info("!!!!!!!!!!!!!!!! Post-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(post_check_results, indent=4))
if any([result["failed"] for result in post_check_results]):
failed_items = json.dumps(
[result for result in new_check_results if result["failed"]],
indent=4)
logger.error("Failed check items:\n{}".format(failed_items))
pytest.fail(
"Post-test sanity check failed with failed items:\n{}".format(
failed_items))
return
logger.info("Done post-test sanity check")
return
def sanity_check(testbed_devices, request):
logger.info("Start pre-test sanity check")
dut = testbed_devices["dut"]
localhost = testbed_devices["localhost"]
skip_sanity = False
allow_recover = False
recover_method = "config_reload"
check_items = set(copy.deepcopy(
constants.SUPPORTED_CHECK_ITEMS)) # Default check items
post_check = False
customized_sanity_check = None
for m in request.node.iter_markers():
logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" %
(m.name, m.args, m.kwargs))
if m.name == "sanity_check":
customized_sanity_check = m
break
if customized_sanity_check:
logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" %
(m.name, str(m.args), str(m.kwargs)))
skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
allow_recover = customized_sanity_check.kwargs.get(
"allow_recover", False)
recover_method = customized_sanity_check.kwargs.get(
"recover_method", "config_reload")
if allow_recover and recover_method not in constants.RECOVER_METHODS:
pytest.warning("Unsupported recover method")
logger.info(
"Fall back to use default recover method 'config_reload'")
recover_method = "config_reload"
check_items = _update_check_items(
check_items, customized_sanity_check.kwargs.get("check_items", []),
constants.SUPPORTED_CHECK_ITEMS)
post_check = customized_sanity_check.kwargs.get("post_check", False)
if request.config.option.allow_recover:
allow_recover = True
logger.info("Sanity check settings: check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \
(check_items, allow_recover, recover_method, post_check))
if skip_sanity:
logger.info(
"Skip sanity check according to configuration of test script.")
yield
return
if not check_items:
logger.info(
"No sanity check item is specified, no pre-test sanity check")
yield
logger.info(
"No sanity check item is specified, no post-test sanity check")
return
print_logs(dut, constants.PRINT_LOGS)
check_results = do_checks(dut, check_items)
logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(check_results, indent=4))
if any([result["failed"] for result in check_results]):
if not allow_recover:
pytest.fail("Pre-test sanity check failed, allow_recover=False")
return
logger.info(
"Pre-test sanity check failed, try to recover, recover_method=%s" %
recover_method)
recover(dut, localhost, recover_method)
logger.info("Run sanity check again after recovery")
new_check_results = do_checks(dut, check_items)
logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(new_check_results, indent=4))
if any([result["failed"] for result in new_check_results]):
pytest.fail(
"Pre-test sanity check failed again after recovered by '%s'" %
recover_method)
return
logger.info("Done pre-test sanity check")
yield
logger.info("Start post-test sanity check")
if not post_check:
logger.info(
"No post-test check is required. Done post-test sanity check")
return
post_check_results = do_checks(dut, check_items)
logger.info("!!!!!!!!!!!!!!!! Post-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
json.dumps(post_check_results, indent=4))
if any([result["failed"] for result in post_check_results]):
pytest.fail("Post-test sanity check failed")
return
logger.info("Done post-test sanity check")
return
def sanity_check(localhost, duthosts, request, fanouthosts, tbinfo):
logger.info("Prepare sanity check")
skip_sanity = False
allow_recover = False
recover_method = "adaptive"
pre_check_items = set(copy.deepcopy(SUPPORTED_CHECKS)) # Default check items
post_check = False
customized_sanity_check = None
for m in request.node.iter_markers():
logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs))
if m.name == "sanity_check":
customized_sanity_check = m
break
if customized_sanity_check:
logger.info("Process marker {} in script. m.args={}, m.kwargs={}"
.format(customized_sanity_check.name, customized_sanity_check.args, customized_sanity_check.kwargs))
skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
allow_recover = customized_sanity_check.kwargs.get("allow_recover", False)
recover_method = customized_sanity_check.kwargs.get("recover_method", "adaptive")
if allow_recover and recover_method not in constants.RECOVER_METHODS:
pytest.warning("Unsupported recover method")
logger.info("Fall back to use default recover method 'config_reload'")
recover_method = "config_reload"
pre_check_items = _update_check_items(
pre_check_items,
customized_sanity_check.kwargs.get("check_items", []),
SUPPORTED_CHECKS)
post_check = customized_sanity_check.kwargs.get("post_check", False)
if request.config.option.skip_sanity:
skip_sanity = True
if skip_sanity:
logger.info("Skip sanity check according to command line argument or configuration of test script.")
yield
return
if request.config.option.allow_recover:
allow_recover = True
if request.config.option.recover_method:
recover_method = request.config.getoption("--recover_method")
if request.config.option.post_check:
post_check = True
cli_check_items = request.config.getoption("--check_items")
cli_post_check_items = request.config.getoption("--post_check_items")
if cli_check_items:
logger.info('Fine tune pre-test check items based on CLI option --check_items')
cli_items_list=str(cli_check_items).split(',')
pre_check_items = _update_check_items(pre_check_items, cli_items_list, SUPPORTED_CHECKS)
pre_check_items = filter_check_items(tbinfo, pre_check_items) # Filter out un-supported checks.
if post_check:
# Prepare post test check items based on the collected pre test check items.
post_check_items = copy.copy(pre_check_items)
if customized_sanity_check:
post_check_items = _update_check_items(
post_check_items,
customized_sanity_check.kwargs.get("post_check_items", []),
SUPPORTED_CHECKS)
if cli_post_check_items:
logger.info('Fine tune post-test check items based on CLI option --post_check_items')
cli_post_items_list = str(cli_post_check_items).split(',')
post_check_items = _update_check_items(post_check_items, cli_post_items_list, SUPPORTED_CHECKS)
post_check_items = filter_check_items(tbinfo, post_check_items) # Filter out un-supported checks.
else:
post_check_items = set()
logger.info("Sanity check settings: skip_sanity=%s, pre_check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s, post_check_items=%s" % \
(skip_sanity, pre_check_items, allow_recover, recover_method, post_check, post_check_items))
for item in pre_check_items.union(post_check_items):
request.fixturenames.append(_item2fixture(item))
# Workaround for pytest requirement.
# Each possibly used check fixture must be executed in setup phase. Otherwise there could be teardown error.
request.getfixturevalue(_item2fixture(item))
if pre_check_items:
logger.info("Start pre-test sanity checks")
# Dynamically attach selected check fixtures to node
for item in set(pre_check_items):
request.fixturenames.append(_item2fixture(item))
print_logs(duthosts)
check_results = do_checks(request, pre_check_items, stage=STAGE_PRE_TEST)
logger.debug("Pre-test sanity check results:\n%s" % json.dumps(check_results, indent=4))
failed_results = [result for result in check_results if result['failed']]
if failed_results:
if not allow_recover:
pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(failed_results, indent=4)))
else:
dut_failed_results = defaultdict(list)
for failed_result in failed_results:
if 'host' in failed_result:
dut_failed_results[failed_result['host']].append(failed_result)
for dut_name, dut_results in dut_failed_results.items():
recover(duthosts[dut_name], localhost, fanouthosts, dut_results, recover_method)
logger.info("Run sanity check again after recovery")
new_check_results = do_checks(request, pre_check_items, stage=STAGE_PRE_TEST, after_recovery=True)
logger.debug("Pre-test sanity check after recovery results:\n%s" % json.dumps(new_check_results, indent=4))
new_failed_results = [result for result in new_check_results if result['failed']]
if new_failed_results:
pt_assert(False, "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(new_failed_results, indent=4)))
logger.info("Done pre-test sanity check")
else:
logger.info('No pre-test sanity check item, skip pre-test sanity check.')
yield
if not post_check:
logger.info("No post-test check is required. Done post-test sanity check")
return
if post_check_items:
logger.info("Start post-test sanity check")
post_check_results = do_checks(request, post_check_items, stage=STAGE_POST_TEST)
logger.debug("Post-test sanity check results:\n%s" % json.dumps(post_check_results, indent=4))
post_failed_results = [result for result in post_check_results if result['failed']]
if post_failed_results:
pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(post_failed_results, indent=4)))
logger.info("Done post-test sanity check")
else:
logger.info('No post-test sanity check item, skip post-test sanity check.')
def sanity_check(localhost, duthosts, request, fanouthosts, tbinfo):
logger.info("Prepare pre-test sanity check")
skip_sanity = False
allow_recover = False
recover_method = "adaptive"
check_items = set(copy.deepcopy(SUPPORTED_CHECKS)) # Default check items
post_check = False
customized_sanity_check = None
for m in request.node.iter_markers():
logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" %
(m.name, m.args, m.kwargs))
if m.name == "sanity_check":
customized_sanity_check = m
break
if customized_sanity_check:
logger.info(
"Process marker {} in script. m.args={}, m.kwargs={}".format(
customized_sanity_check.name, customized_sanity_check.args,
customized_sanity_check.kwargs))
skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
allow_recover = customized_sanity_check.kwargs.get(
"allow_recover", False)
recover_method = customized_sanity_check.kwargs.get(
"recover_method", "adaptive")
if allow_recover and recover_method not in constants.RECOVER_METHODS:
pytest.warning("Unsupported recover method")
logger.info(
"Fall back to use default recover method 'config_reload'")
recover_method = "config_reload"
check_items = _update_check_items(
check_items, customized_sanity_check.kwargs.get("check_items", []),
SUPPORTED_CHECKS)
post_check = customized_sanity_check.kwargs.get("post_check", False)
if request.config.option.skip_sanity:
skip_sanity = True
if skip_sanity:
logger.info(
"Skip sanity check according to command line argument or configuration of test script."
)
yield
return
if request.config.option.allow_recover:
allow_recover = True
cli_items = request.config.getoption("--check_items")
if cli_items:
cli_items_list = str(cli_items).split(',')
check_items = _update_check_items(check_items, cli_items_list,
SUPPORTED_CHECKS)
# ignore BGP check for particular topology type
if tbinfo['topo']['type'] == 'ptf' and 'bgp' in check_items:
check_items.remove('bgp')
if 'dualtor' not in tbinfo['topo']['name']:
check_items.remove('mux_simulator')
logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \
(skip_sanity, check_items, allow_recover, recover_method, post_check))
if not check_items:
logger.info(
"No sanity check item is specified, no pre-test sanity check")
yield
logger.info(
"No sanity check item is specified, no post-test sanity check")
return
# Dynamically attach selected check fixtures to node
for item in check_items:
request.fixturenames.append(_item2fixture(item))
print_logs(duthosts)
logger.info("Start pre-test sanity checks")
check_results = do_checks(request, check_items)
logger.debug("Pre-test sanity check results:\n%s" %
json.dumps(check_results, indent=4))
failed_results = [result for result in check_results if result['failed']]
if failed_results:
if not allow_recover:
pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(failed_results, indent=4)))
else:
dut_failed_results = defaultdict(list)
for failed_result in failed_results:
if 'host' in failed_result:
dut_failed_results[failed_result['host']].append(
failed_result)
for dut_name, dut_results in dut_failed_results.items():
recover(duthosts[dut_name], localhost, fanouthosts,
dut_results, recover_method)
logger.info("Run sanity check again after recovery")
new_check_results = do_checks(request, check_items)
logger.debug("Pre-test sanity check after recovery results:\n%s" %
json.dumps(new_check_results, indent=4))
new_failed_results = [
result for result in new_check_results if result['failed']
]
if new_failed_results:
pt_assert(False, "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(new_failed_results, indent=4)))
logger.info("Done pre-test sanity check")
yield
if not post_check:
logger.info(
"No post-test check is required. Done post-test sanity check")
return
logger.info("Start post-test sanity check")
post_check_results = do_checks(request, check_items)
logger.debug("Post-test sanity check results:\n%s" %
json.dumps(post_check_results, indent=4))
post_failed_results = [
result for result in post_check_results if result['failed']
]
if post_failed_results:
pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\
.format(json.dumps(post_failed_results, indent=4)))
logger.info("Done post-test sanity check")
return
