def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir): """Parameterized test. :param instance: test instance. Should not be changed during test run :param executable: timetest executable to run :param niter: number of times to run executable :param cl_cache_dir: directory to store OpenCL cache :param test_info: custom `test_info` field of built-in `request` pytest fixture :param temp_dir: path to a temporary directory. Will be cleaned up after test run """ # Prepare model to get model_path model_path = instance["model"].get("path") assert model_path, "Model path is empty" model_path = Path(expand_env_vars(model_path)) # Copy model to a local temporary directory model_dir = temp_dir / "model" shutil.copytree(model_path.parent, model_dir) model_path = model_dir / model_path.name # Run executable exe_args = { "executable": Path(executable), "model": Path(model_path), "device": instance["device"]["name"], "niter": niter } if exe_args["device"] == "GPU": # Generate cl_cache via additional timetest run _exe_args = exe_args.copy() _exe_args["niter"] = 1 logging.info("Run timetest once to generate cl_cache to {}".format( cl_cache_dir)) run_timetest(_exe_args, log=logging) assert os.listdir(cl_cache_dir), "cl_cache isn't generated" retcode, aggr_stats = run_timetest(exe_args, log=logging) assert retcode == 0, "Run of executable failed" # Add timetest results to submit to database and save in new test conf as references test_info["results"] = aggr_stats # Compare with references comparison_status = 0 for step_name, references in instance["references"].items(): for metric, reference_val in references.items(): if aggr_stats[step_name][metric] > reference_val * REFS_FACTOR: logging.error( "Comparison failed for '{}' step for '{}' metric. Reference: {}. Current values: {}" .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) comparison_status = 1 else: logging.info( "Comparison passed for '{}' step for '{}' metric. Reference: {}. Current values: {}" .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) assert comparison_status == 0, "Comparison with references failed"
def pytest_generate_tests(metafunc): """Pytest hook for test generation. Generate parameterized tests from discovered modules and test config parameters. """ with open(metafunc.config.getoption('test_conf'), "r") as file: test_cases = expand_env_vars(yaml.safe_load(file)) if test_cases: metafunc.parametrize("instance", test_cases)
def test_timetest(instance, executable, niter, cl_cache_dir): """Parameterized test. :param instance: test instance :param executable: timetest executable to run :param niter: number of times to run executable """ # Prepare model to get model_path model_path = instance["model"].get("path") assert model_path, "Model path is empty" # Run executable exe_args = { "executable": Path(executable), "model": Path(expand_env_vars(model_path)), "device": instance["device"]["name"], "niter": niter } if exe_args["device"] == "GPU": # Generate cl_cache via additional timetest run _exe_args = exe_args.copy() _exe_args["niter"] = 1 logging.info("Run timetest once to generate cl_cache to {}".format( cl_cache_dir)) run_timetest(_exe_args, log=logging) assert os.listdir(cl_cache_dir), "cl_cache isn't generated" retcode, aggr_stats = run_timetest(exe_args, log=logging) assert retcode == 0, "Run of executable failed" # Add timetest results to submit to database and save in new test conf as references instance["results"] = aggr_stats # Compare with references comparison_status = 0 for step_name, references in instance["references"].items(): for metric, reference_val in references.items(): if aggr_stats[step_name][metric] > reference_val * REFS_FACTOR: logging.error( "Comparison failed for '{}' step for '{}' metric. Reference: {}. Current values: {}" .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) comparison_status = 1 else: logging.info( "Comparison passed for '{}' step for '{}' metric. Reference: {}. Current values: {}" .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) assert comparison_status == 0, "Comparison with references failed"
def test_timetest(instance, executable, niter, cl_cache_dir, model_cache_dir, test_info, temp_dir, validate_test_case, prepare_db_info): """Parameterized test. :param instance: test instance. Should not be changed during test run :param executable: timetest executable to run :param niter: number of times to run executable :param cl_cache_dir: directory to store OpenCL cache :param model_cache_dir: directory to store IE model cache :param test_info: custom `test_info` field of built-in `request` pytest fixture :param temp_dir: path to a temporary directory. Will be cleaned up after test run :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled :param prepare_db_info: custom pytest fixture. Should be declared as test argument to be enabled """ # Prepare model to get model_path model_path = instance["model"].get("path") assert model_path, "Model path is empty" model_path = Path(expand_env_vars(model_path)) # Copy model to a local temporary directory model_dir = temp_dir / "model" shutil.copytree(model_path.parent, model_dir) model_path = model_dir / model_path.name # Run executable exe_args = { "executable": Path(executable), "model": Path(model_path), "device": instance["device"]["name"], "niter": niter } logging.info("Run timetest once to generate any cache") retcode, msg, _, _ = run_timetest({**exe_args, "niter": 1}, log=logging) assert retcode == 0, f"Run of executable for warm up failed: {msg}" if cl_cache_dir: assert os.listdir(cl_cache_dir), "cl_cache isn't generated" if model_cache_dir: assert os.listdir(model_cache_dir), "model_cache isn't generated" retcode, msg, aggr_stats, raw_stats = run_timetest(exe_args, log=logging) assert retcode == 0, f"Run of executable failed: {msg}" # Add timetest results to submit to database and save in new test conf as references test_info["results"] = aggr_stats test_info["raw_results"] = raw_stats