def sanity_check(localhost, duthost, request, fanouthosts, tbinfo): logger.info("Start pre-test sanity check") skip_sanity = False allow_recover = False recover_method = "adaptive" check_items = set(copy.deepcopy( constants.DEFAULT_CHECK_ITEMS)) # Default check items post_check = False customized_sanity_check = None for m in request.node.iter_markers(): logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs)) if m.name == "sanity_check": customized_sanity_check = m break if customized_sanity_check: logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" % (m.name, str(m.args), str(m.kwargs))) skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False) allow_recover = customized_sanity_check.kwargs.get( "allow_recover", False) recover_method = customized_sanity_check.kwargs.get( "recover_method", "adaptive") if allow_recover and recover_method not in constants.RECOVER_METHODS: pytest.warning("Unsupported recover method") logger.info( "Fall back to use default recover method 'config_reload'") recover_method = "config_reload" check_items = _update_check_items( check_items, customized_sanity_check.kwargs.get("check_items", []), constants.SUPPORTED_CHECK_ITEMS) post_check = customized_sanity_check.kwargs.get("post_check", False) if request.config.option.skip_sanity: skip_sanity = True if request.config.option.allow_recover: allow_recover = True items = request.config.getoption("--check_items") if items: items_array = str(items).split(',') check_items = _update_check_items(check_items, items_array, constants.SUPPORTED_CHECK_ITEMS) # ignore BGP check for particular topology type if tbinfo['topo']['type'] == 'ptf' and 'bgp' in check_items: check_items.remove('bgp') logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \ (skip_sanity, check_items, allow_recover, recover_method, post_check)) if skip_sanity: logger.info( "Skip sanity check according to command line argument or configuration of test script." ) yield return if not check_items: logger.info( "No sanity check item is specified, no pre-test sanity check") yield logger.info( "No sanity check item is specified, no post-test sanity check") return print_logs(duthost, constants.PRINT_LOGS) check_results = do_checks(duthost, check_items) logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(check_results, indent=4)) if any([result["failed"] for result in check_results]): if not allow_recover: pytest.fail( "Pre-test sanity check failed, allow_recover=False {}".format( check_results)) return logger.info( "Pre-test sanity check failed, try to recover, recover_method=%s" % recover_method) recover(duthost, localhost, fanouthosts, check_results, recover_method) logger.info("Run sanity check again after recovery") new_check_results = do_checks(duthost, check_items) logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(new_check_results, indent=4)) if any([result["failed"] for result in new_check_results]): failed_items = json.dumps( [result for result in new_check_results if result["failed"]], indent=4) logger.error("Failed check items:\n{}".format(failed_items)) pytest.fail( "Pre-test sanity check failed again after recovered by '{}' with failed items:\n{}" .format(recover_method, failed_items)) return logger.info("Done pre-test sanity check") yield logger.info("Start post-test sanity check") if not post_check: logger.info( "No post-test check is required. Done post-test sanity check") return post_check_results = do_checks(duthost, check_items) logger.info("!!!!!!!!!!!!!!!! Post-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(post_check_results, indent=4)) if any([result["failed"] for result in post_check_results]): failed_items = json.dumps( [result for result in new_check_results if result["failed"]], indent=4) logger.error("Failed check items:\n{}".format(failed_items)) pytest.fail( "Post-test sanity check failed with failed items:\n{}".format( failed_items)) return logger.info("Done post-test sanity check") return
def sanity_check(testbed_devices, request): logger.info("Start pre-test sanity check") dut = testbed_devices["dut"] localhost = testbed_devices["localhost"] skip_sanity = False allow_recover = False recover_method = "config_reload" check_items = set(copy.deepcopy( constants.SUPPORTED_CHECK_ITEMS)) # Default check items post_check = False customized_sanity_check = None for m in request.node.iter_markers(): logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs)) if m.name == "sanity_check": customized_sanity_check = m break if customized_sanity_check: logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" % (m.name, str(m.args), str(m.kwargs))) skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False) allow_recover = customized_sanity_check.kwargs.get( "allow_recover", False) recover_method = customized_sanity_check.kwargs.get( "recover_method", "config_reload") if allow_recover and recover_method not in constants.RECOVER_METHODS: pytest.warning("Unsupported recover method") logger.info( "Fall back to use default recover method 'config_reload'") recover_method = "config_reload" check_items = _update_check_items( check_items, customized_sanity_check.kwargs.get("check_items", []), constants.SUPPORTED_CHECK_ITEMS) post_check = customized_sanity_check.kwargs.get("post_check", False) if request.config.option.allow_recover: allow_recover = True logger.info("Sanity check settings: check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \ (check_items, allow_recover, recover_method, post_check)) if skip_sanity: logger.info( "Skip sanity check according to configuration of test script.") yield return if not check_items: logger.info( "No sanity check item is specified, no pre-test sanity check") yield logger.info( "No sanity check item is specified, no post-test sanity check") return print_logs(dut, constants.PRINT_LOGS) check_results = do_checks(dut, check_items) logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(check_results, indent=4)) if any([result["failed"] for result in check_results]): if not allow_recover: pytest.fail("Pre-test sanity check failed, allow_recover=False") return logger.info( "Pre-test sanity check failed, try to recover, recover_method=%s" % recover_method) recover(dut, localhost, recover_method) logger.info("Run sanity check again after recovery") new_check_results = do_checks(dut, check_items) logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(new_check_results, indent=4)) if any([result["failed"] for result in new_check_results]): pytest.fail( "Pre-test sanity check failed again after recovered by '%s'" % recover_method) return logger.info("Done pre-test sanity check") yield logger.info("Start post-test sanity check") if not post_check: logger.info( "No post-test check is required. Done post-test sanity check") return post_check_results = do_checks(dut, check_items) logger.info("!!!!!!!!!!!!!!!! Post-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \ json.dumps(post_check_results, indent=4)) if any([result["failed"] for result in post_check_results]): pytest.fail("Post-test sanity check failed") return logger.info("Done post-test sanity check") return
def sanity_check(localhost, duthosts, request, fanouthosts, tbinfo): logger.info("Prepare sanity check") skip_sanity = False allow_recover = False recover_method = "adaptive" pre_check_items = set(copy.deepcopy(SUPPORTED_CHECKS)) # Default check items post_check = False customized_sanity_check = None for m in request.node.iter_markers(): logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs)) if m.name == "sanity_check": customized_sanity_check = m break if customized_sanity_check: logger.info("Process marker {} in script. m.args={}, m.kwargs={}" .format(customized_sanity_check.name, customized_sanity_check.args, customized_sanity_check.kwargs)) skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False) allow_recover = customized_sanity_check.kwargs.get("allow_recover", False) recover_method = customized_sanity_check.kwargs.get("recover_method", "adaptive") if allow_recover and recover_method not in constants.RECOVER_METHODS: pytest.warning("Unsupported recover method") logger.info("Fall back to use default recover method 'config_reload'") recover_method = "config_reload" pre_check_items = _update_check_items( pre_check_items, customized_sanity_check.kwargs.get("check_items", []), SUPPORTED_CHECKS) post_check = customized_sanity_check.kwargs.get("post_check", False) if request.config.option.skip_sanity: skip_sanity = True if skip_sanity: logger.info("Skip sanity check according to command line argument or configuration of test script.") yield return if request.config.option.allow_recover: allow_recover = True if request.config.option.recover_method: recover_method = request.config.getoption("--recover_method") if request.config.option.post_check: post_check = True cli_check_items = request.config.getoption("--check_items") cli_post_check_items = request.config.getoption("--post_check_items") if cli_check_items: logger.info('Fine tune pre-test check items based on CLI option --check_items') cli_items_list=str(cli_check_items).split(',') pre_check_items = _update_check_items(pre_check_items, cli_items_list, SUPPORTED_CHECKS) pre_check_items = filter_check_items(tbinfo, pre_check_items) # Filter out un-supported checks. if post_check: # Prepare post test check items based on the collected pre test check items. post_check_items = copy.copy(pre_check_items) if customized_sanity_check: post_check_items = _update_check_items( post_check_items, customized_sanity_check.kwargs.get("post_check_items", []), SUPPORTED_CHECKS) if cli_post_check_items: logger.info('Fine tune post-test check items based on CLI option --post_check_items') cli_post_items_list = str(cli_post_check_items).split(',') post_check_items = _update_check_items(post_check_items, cli_post_items_list, SUPPORTED_CHECKS) post_check_items = filter_check_items(tbinfo, post_check_items) # Filter out un-supported checks. else: post_check_items = set() logger.info("Sanity check settings: skip_sanity=%s, pre_check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s, post_check_items=%s" % \ (skip_sanity, pre_check_items, allow_recover, recover_method, post_check, post_check_items)) for item in pre_check_items.union(post_check_items): request.fixturenames.append(_item2fixture(item)) # Workaround for pytest requirement. # Each possibly used check fixture must be executed in setup phase. Otherwise there could be teardown error. request.getfixturevalue(_item2fixture(item)) if pre_check_items: logger.info("Start pre-test sanity checks") # Dynamically attach selected check fixtures to node for item in set(pre_check_items): request.fixturenames.append(_item2fixture(item)) print_logs(duthosts) check_results = do_checks(request, pre_check_items, stage=STAGE_PRE_TEST) logger.debug("Pre-test sanity check results:\n%s" % json.dumps(check_results, indent=4)) failed_results = [result for result in check_results if result['failed']] if failed_results: if not allow_recover: pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(failed_results, indent=4))) else: dut_failed_results = defaultdict(list) for failed_result in failed_results: if 'host' in failed_result: dut_failed_results[failed_result['host']].append(failed_result) for dut_name, dut_results in dut_failed_results.items(): recover(duthosts[dut_name], localhost, fanouthosts, dut_results, recover_method) logger.info("Run sanity check again after recovery") new_check_results = do_checks(request, pre_check_items, stage=STAGE_PRE_TEST, after_recovery=True) logger.debug("Pre-test sanity check after recovery results:\n%s" % json.dumps(new_check_results, indent=4)) new_failed_results = [result for result in new_check_results if result['failed']] if new_failed_results: pt_assert(False, "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(new_failed_results, indent=4))) logger.info("Done pre-test sanity check") else: logger.info('No pre-test sanity check item, skip pre-test sanity check.') yield if not post_check: logger.info("No post-test check is required. Done post-test sanity check") return if post_check_items: logger.info("Start post-test sanity check") post_check_results = do_checks(request, post_check_items, stage=STAGE_POST_TEST) logger.debug("Post-test sanity check results:\n%s" % json.dumps(post_check_results, indent=4)) post_failed_results = [result for result in post_check_results if result['failed']] if post_failed_results: pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(post_failed_results, indent=4))) logger.info("Done post-test sanity check") else: logger.info('No post-test sanity check item, skip post-test sanity check.')
def sanity_check(localhost, duthosts, request, fanouthosts, tbinfo): logger.info("Prepare pre-test sanity check") skip_sanity = False allow_recover = False recover_method = "adaptive" check_items = set(copy.deepcopy(SUPPORTED_CHECKS)) # Default check items post_check = False customized_sanity_check = None for m in request.node.iter_markers(): logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs)) if m.name == "sanity_check": customized_sanity_check = m break if customized_sanity_check: logger.info( "Process marker {} in script. m.args={}, m.kwargs={}".format( customized_sanity_check.name, customized_sanity_check.args, customized_sanity_check.kwargs)) skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False) allow_recover = customized_sanity_check.kwargs.get( "allow_recover", False) recover_method = customized_sanity_check.kwargs.get( "recover_method", "adaptive") if allow_recover and recover_method not in constants.RECOVER_METHODS: pytest.warning("Unsupported recover method") logger.info( "Fall back to use default recover method 'config_reload'") recover_method = "config_reload" check_items = _update_check_items( check_items, customized_sanity_check.kwargs.get("check_items", []), SUPPORTED_CHECKS) post_check = customized_sanity_check.kwargs.get("post_check", False) if request.config.option.skip_sanity: skip_sanity = True if skip_sanity: logger.info( "Skip sanity check according to command line argument or configuration of test script." ) yield return if request.config.option.allow_recover: allow_recover = True cli_items = request.config.getoption("--check_items") if cli_items: cli_items_list = str(cli_items).split(',') check_items = _update_check_items(check_items, cli_items_list, SUPPORTED_CHECKS) # ignore BGP check for particular topology type if tbinfo['topo']['type'] == 'ptf' and 'bgp' in check_items: check_items.remove('bgp') if 'dualtor' not in tbinfo['topo']['name']: check_items.remove('mux_simulator') logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \ (skip_sanity, check_items, allow_recover, recover_method, post_check)) if not check_items: logger.info( "No sanity check item is specified, no pre-test sanity check") yield logger.info( "No sanity check item is specified, no post-test sanity check") return # Dynamically attach selected check fixtures to node for item in check_items: request.fixturenames.append(_item2fixture(item)) print_logs(duthosts) logger.info("Start pre-test sanity checks") check_results = do_checks(request, check_items) logger.debug("Pre-test sanity check results:\n%s" % json.dumps(check_results, indent=4)) failed_results = [result for result in check_results if result['failed']] if failed_results: if not allow_recover: pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(failed_results, indent=4))) else: dut_failed_results = defaultdict(list) for failed_result in failed_results: if 'host' in failed_result: dut_failed_results[failed_result['host']].append( failed_result) for dut_name, dut_results in dut_failed_results.items(): recover(duthosts[dut_name], localhost, fanouthosts, dut_results, recover_method) logger.info("Run sanity check again after recovery") new_check_results = do_checks(request, check_items) logger.debug("Pre-test sanity check after recovery results:\n%s" % json.dumps(new_check_results, indent=4)) new_failed_results = [ result for result in new_check_results if result['failed'] ] if new_failed_results: pt_assert(False, "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(new_failed_results, indent=4))) logger.info("Done pre-test sanity check") yield if not post_check: logger.info( "No post-test check is required. Done post-test sanity check") return logger.info("Start post-test sanity check") post_check_results = do_checks(request, check_items) logger.debug("Post-test sanity check results:\n%s" % json.dumps(post_check_results, indent=4)) post_failed_results = [ result for result in post_check_results if result['failed'] ] if post_failed_results: pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"\ .format(json.dumps(post_failed_results, indent=4))) logger.info("Done post-test sanity check") return