def pytest_generate_tests(metafunc): # Load all the JSON files in the directory dir_path = os.path.dirname(os.path.realpath(__file__)) expectation_dirs = [ dir_ for dir_ in os.listdir(dir_path) if os.path.isdir(os.path.join(dir_path, dir_)) ] parametrized_tests = [] ids = [] backends = build_test_backends_list_cfe(metafunc) validator_with_data = None for expectation_category in expectation_dirs: test_configuration_files = glob.glob(dir_path + "/" + expectation_category + "/*.json") for c in backends: for filename in test_configuration_files: file = open(filename) test_configuration = json.load(file) for d in test_configuration["datasets"]: datasets = [] # optional only_for and suppress_test flag at the datasets-level that can prevent data being # added to incompatible backends. Currently only used by expect_column_values_to_be_unique only_for = d.get("only_for") if only_for and not isinstance(only_for, list): # coerce into list if passed in as string only_for = [only_for] suppress_test_for = d.get("suppress_test_for") if suppress_test_for and not isinstance( suppress_test_for, list): # coerce into list if passed in as string suppress_test_for = [suppress_test_for] if candidate_test_is_on_temporary_notimplemented_list_cfe( c, test_configuration["expectation_type"]): skip_expectation = True elif suppress_test_for and c in suppress_test_for: continue elif only_for and c not in only_for: continue else: skip_expectation = False if isinstance(d["data"], list): sqlite_db_path = generate_sqlite_db_path() for dataset in d["data"]: datasets.append( get_test_validator_with_data( c, dataset["data"], dataset.get("schemas"), table_name=dataset.get("dataset_name"), sqlite_db_path=sqlite_db_path, )) validator_with_data = datasets[0] else: schemas = d["schemas"] if "schemas" in d else None validator_with_data = get_test_validator_with_data( c, d["data"], schemas=schemas) for test in d["tests"]: generate_test = True skip_test = False only_for = test.get("only_for") if only_for: # if we're not on the "only_for" list, then never even generate the test generate_test = False if not isinstance(only_for, list): # coerce into list if passed in as string only_for = [only_for] if validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ): # Call out supported dialects if "sqlalchemy" in only_for: generate_test = True elif ("sqlite" in only_for and sqliteDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, sqliteDialect, )): generate_test = True elif ("postgresql" in only_for and postgresqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, postgresqlDialect, )): generate_test = True elif ("mysql" in only_for and mysqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mysqlDialect, )): generate_test = True elif ("mssql" in only_for and mssqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mssqlDialect, )): generate_test = True elif ("bigquery" in only_for and BigQueryDialect is not None and hasattr( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, "name", ) and validator_with_data.execution_engine. active_batch_data.sql_engine_dialect.name == "bigquery"): generate_test = True elif ("trino" in test["only_for"] and BigQueryDialect is not None and hasattr( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, "name", ) and validator_with_data.execution_engine. active_batch_data.sql_engine_dialect.name == "trino"): generate_test = True elif validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, PandasBatchData, ): major, minor, *_ = pd.__version__.split(".") if "pandas" in only_for: generate_test = True if (("pandas_022" in only_for or "pandas_023" in only_for) and major == "0" and minor in ["22", "23"]): generate_test = True if ("pandas>=024" in only_for) and ( (major == "0" and int(minor) >= 24) or int(major) >= 1): generate_test = True elif validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SparkDFBatchData, ): if "spark" in only_for: generate_test = True if not generate_test: continue suppress_test_for = test.get("suppress_test_for") if suppress_test_for: if not isinstance(suppress_test_for, list): # coerce into list if passed in as string suppress_test_for = [suppress_test_for] if (("sqlalchemy" in suppress_test_for and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, )) or ("sqlite" in suppress_test_for and sqliteDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, sqliteDialect, )) or ("postgresql" in suppress_test_for and postgresqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, postgresqlDialect, )) or ("mysql" in suppress_test_for and mysqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mysqlDialect, )) or ("mssql" in suppress_test_for and mssqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mssqlDialect, )) or ("bigquery" in suppress_test_for and BigQueryDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and hasattr( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, "name", ) and validator_with_data.execution_engine. active_batch_data.sql_engine_dialect.name == "bigquery") or ("trino" in suppress_test_for and trinoDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and hasattr( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, "name", ) and validator_with_data.execution_engine. active_batch_data.sql_engine_dialect.name == "trino") or ("pandas" in suppress_test_for and validator_with_data and isinstance( validator_with_data. execution_engine.active_batch_data, PandasBatchData, )) or ("spark" in suppress_test_for and validator_with_data and isinstance( validator_with_data. execution_engine.active_batch_data, SparkDFBatchData, ))): skip_test = True # Known condition: SqlAlchemy does not support allow_cross_type_comparisons if ("allow_cross_type_comparisons" in test["in"] and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, )): skip_test = True parametrized_tests.append({ "expectation_type": test_configuration["expectation_type"], "validator_with_data": validator_with_data, "test": test, "skip": skip_expectation or skip_test, }) ids.append(c + "/" + expectation_category + "/" + test_configuration["expectation_type"] + ":" + test["title"]) metafunc.parametrize("test_case", parametrized_tests, ids=ids)
def pytest_generate_tests(metafunc): # Load all the JSON files in the directory dir_path = os.path.dirname(os.path.realpath(__file__)) expectation_dirs = [ dir_ for dir_ in os.listdir(dir_path) if os.path.isdir(os.path.join(dir_path, dir_)) ] parametrized_tests = [] ids = [] backends = build_test_backends_list_cfe(metafunc) for expectation_category in expectation_dirs: test_configuration_files = glob.glob(dir_path + "/" + expectation_category + "/*.json") for c in backends: for filename in test_configuration_files: file = open(filename) test_configuration = json.load(file) for d in test_configuration["datasets"]: datasets = [] if candidate_test_is_on_temporary_notimplemented_list_cfe( c, test_configuration["expectation_type"]): skip_expectation = True schemas = validator_with_data = None else: skip_expectation = False if isinstance(d["data"], list): sqlite_db_path = os.path.abspath( os.path.join( tmp_dir, "sqlite_db" + "".join([ random.choice(string.ascii_letters + string.digits) for _ in range(8) ]) + ".db", )) for dataset in d["data"]: datasets.append( get_test_validator_with_data( c, dataset["data"], dataset.get("schemas"), table_name=dataset.get("dataset_name"), sqlite_db_path=sqlite_db_path, )) validator_with_data = datasets[0] else: schemas = d["schemas"] if "schemas" in d else None validator_with_data = get_test_validator_with_data( c, d["data"], schemas=schemas) for test in d["tests"]: generate_test = True skip_test = False if "only_for" in test: # if we're not on the "only_for" list, then never even generate the test generate_test = False if not isinstance(test["only_for"], list): raise ValueError("Invalid test specification.") if validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ): # Call out supported dialects if "sqlalchemy" in test["only_for"]: generate_test = True elif ("sqlite" in test["only_for"] and sqliteDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, sqliteDialect, )): generate_test = True elif ("postgresql" in test["only_for"] and postgresqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, postgresqlDialect, )): generate_test = True elif ("mysql" in test["only_for"] and mysqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mysqlDialect, )): generate_test = True elif ("mssql" in test["only_for"] and mssqlDialect is not None and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mssqlDialect, )): generate_test = True elif validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, PandasBatchData, ): if "pandas" in test["only_for"]: generate_test = True if ("pandas_022" in test["only_for"] or "pandas_023" in test["only_for"]) and int( pd.__version__.split(".")[1]) in [ 22, 23 ]: generate_test = True if ("pandas>=24" in test["only_for"]) and int( pd.__version__.split(".")[1]) > 24: generate_test = True elif validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SparkDFBatchData, ): if "spark" in test["only_for"]: generate_test = True if not generate_test: continue if "suppress_test_for" in test and ( ("sqlalchemy" in test["suppress_test_for"] and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, )) or ("sqlite" in test["suppress_test_for"] and sqliteDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, sqliteDialect, )) or ("postgresql" in test["suppress_test_for"] and postgresqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, postgresqlDialect, )) or ("mysql" in test["suppress_test_for"] and mysqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mysqlDialect, )) or ("mssql" in test["suppress_test_for"] and mssqlDialect is not None and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, ) and isinstance( validator_with_data.execution_engine. active_batch_data.sql_engine_dialect, mssqlDialect, )) or ("pandas" in test["suppress_test_for"] and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, PandasBatchData, )) or ("spark" in test["suppress_test_for"] and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SparkDFBatchData, ))): skip_test = True # Known condition: SqlAlchemy does not support allow_cross_type_comparisons if ("allow_cross_type_comparisons" in test["in"] and validator_with_data and isinstance( validator_with_data.execution_engine. active_batch_data, SqlAlchemyBatchData, )): skip_test = True parametrized_tests.append({ "expectation_type": test_configuration["expectation_type"], "validator_with_data": validator_with_data, "test": test, "skip": skip_expectation or skip_test, }) ids.append(c + "/" + expectation_category + "/" + test_configuration["expectation_type"] + ":" + test["title"]) metafunc.parametrize("test_case", parametrized_tests, ids=ids)