def pytest_generate_tests(metafunc): """Generate tests depending on command line options.""" params = [] ids = [] with open(metafunc.config.getoption("test_conf"), "r") as file: test_cases = yaml.safe_load(file) for test in test_cases: model_list = [] test_id_list = [] for model in test: extra_args = {} if "marks" in test: extra_args["marks"] = test["marks"] model = model["model"] is_omz = model.get("type") == "omz" if is_omz: test_id_list.append(f'{model["name"]}_{model["precision"]}') else: test_id_list.append(model["path"].split("/")[-1]) model["path"] = Path(expand_env_vars(model["path"])) model_list.append(model) ids = ids + ['-'.join(test_id_list)] params.append(pytest.param('-'.join(test_id_list), model_list), **extra_args) metafunc.parametrize("test_id, models", params, ids=ids)
def test_timetest(instance, executable, niter, cl_cache_dir, cpu_cache, vpu_compiler, perf_hint, model_cache_dir, test_info, temp_dir, validate_test_case, prepare_db_info): """Parameterized test. :param instance: test instance. Should not be changed during test run :param executable: timetest executable to run :param niter: number of times to run executable :param cl_cache_dir: directory to store OpenCL cache :param cpu_cache: flag to enable model CPU cache :param vpu_compiler: flag to change VPUX compiler type :param perf_hint: performance hint (optimize device for latency or throughput settings) :param model_cache_dir: directory to store IE model cache :param test_info: custom `test_info` field of built-in `request` pytest fixture :param temp_dir: path to a temporary directory. Will be cleaned up after test run :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled :param prepare_db_info: custom pytest fixture. Should be declared as test argument to be enabled """ # Prepare model to get model_path model_path = instance["model"].get("path") assert model_path, "Model path is empty" model_path = Path(expand_env_vars(model_path)) # Copy model to a local temporary directory model_dir = temp_dir / "model" shutil.copytree(model_path.parent, model_dir) model_path = model_dir / model_path.name # Run executable exe_args = { "executable": Path(executable), "model": Path(model_path), "device": instance["device"]["name"], "niter": niter, "perf_hint": perf_hint, "cpu_cache": cpu_cache, "vpu_compiler": vpu_compiler if vpu_compiler else "" } logging.info("Run timetest once to generate any cache") retcode, msg, _, _ = run_timetest({**exe_args, "niter": 1}, log=logging) assert retcode == 0, f"Run of executable for warm up failed: {msg}" if cl_cache_dir: assert os.listdir(cl_cache_dir), "cl_cache isn't generated" if model_cache_dir: assert os.listdir(model_cache_dir), "model_cache isn't generated" retcode, msg, aggr_stats, raw_stats = run_timetest(exe_args, log=logging) assert retcode == 0, f"Run of executable failed: {msg}" # Add timetest results to submit to database and save in new test conf as references test_info["results"] = aggr_stats test_info["raw_results"] = raw_stats
def test(instance, executable, niter, temp_dir, omz_models_conversion, validate_test_case, prepare_db_info): """Parameterized test. :param instance: test instance. Should not be changed during test run :param executable: test executable to run :param niter: number of times to run executable :param temp_dir: path to a temporary directory. Will be cleaned up after test run :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled :param prepare_db_info: custom pytest fixture. Should be declared as test argument to be enabled :param omz_models_conversion: custom pytest fixture. Should be declared as test argument to be enabled """ # Prepare model to get model_path model_path = '' cache_model_path = instance["instance"]["model"].get("cache_path") irs_model_path = instance["instance"]["model"].get("irs_out_path") if os.path.isfile(irs_model_path): model_path = irs_model_path elif os.path.isfile(cache_model_path): model_path = cache_model_path assert model_path, "Model path is empty" model_path = Path(expand_env_vars(model_path)) # Copy model to a local temporary directory model_dir = temp_dir / "model" shutil.copytree(model_path.parent, model_dir) model_path = model_dir / model_path.name # Run executable exe_args = { "executable": Path(executable), "model": Path(model_path), "device": instance["instance"]["device"]["name"], "niter": niter } retcode, msg, aggr_stats, raw_stats = run_memorytest(exe_args, log=logging) assert retcode == 0, f"Run of executable failed: {msg}" # Add test results to submit to database and save in new test conf as references instance["results"] = aggr_stats instance["raw_results"] = raw_stats # Compare with references metrics_comparator_status = compare_with_references( aggr_stats, instance["orig_instance"]["references"]) assert metrics_comparator_status == 0, "Comparison with references failed"
def pytest_generate_tests(metafunc): """Generate tests depending on command line options.""" params = [] ids = [] with open(metafunc.config.getoption("test_conf"), "r") as file: test_cases = yaml.safe_load(file) for test in test_cases: extra_args = {} model_path = test["model"]["path"] if "marks" in test: extra_args["marks"] = test["marks"] test_id = model_path.replace("$", "").replace("{", "").replace("}", "") params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args)) ids = ids + [test_id] metafunc.parametrize("test_id, model", params, ids=ids)
def pytest_generate_tests(metafunc): """ Generate tests depending on command line options """ params = [] ids = [] with open(metafunc.config.getoption('test_conf'), "r") as file: test_cases = yaml.safe_load(file) for test in test_cases: extra_args = {} model_path = test["model"]["path"] if "marks" in test: extra_args["marks"] = test["marks"] params.append( pytest.param(Path(expand_env_vars(model_path)), **extra_args)) ids = ids + [model_path] metafunc.parametrize("model", params, ids=ids)
def pytest_generate_tests(metafunc): """Generate tests depending on command line options.""" params = [] ids = [] with open(metafunc.config.getoption("test_conf"), "r") as file: test_cases = yaml.safe_load(file) for test in test_cases: model_list = [] test_id_list = [] for models in test: extra_args = {} model_path = models["model"]["path"] if "marks" in test: extra_args["marks"] = test["marks"] model_list.append(expand_env_vars(model_path)) test_id_list.append(model_path.split("/")[- 1]) ids = ids + ['-'.join(test_id_list)] params.append(pytest.param('-'.join(test_id_list), model_list), **extra_args) metafunc.parametrize("test_id, models", params, ids=ids)