def test_expect_column_values_to_be_of_type_string_dialect_sqlite(sa): df = pd.DataFrame({"col": ["test_val1", "test_val2"]}) validator = build_sa_validator_with_data(df, "sqlite") result = validator.expect_column_values_to_be_of_type("col", type_="TEXT") assert result == ExpectationValidationResult( success=True, expectation_config={ "expectation_type": "expect_column_values_to_be_of_type", "kwargs": { "column": "col", "type_": "TEXT", }, "meta": {}, }, result={ "element_count": 2, "unexpected_count": 0, "unexpected_percent": 0.0, "partial_unexpected_list": [], "missing_count": 0, "missing_percent": 0.0, "unexpected_percent_total": 0.0, "unexpected_percent_nonmissing": 0.0, }, exception_info={ "raised_exception": False, "exception_traceback": None, "exception_message": None, }, meta={}, )
def test_sa_expect_column_value_z_scores_to_be_less_than_impl( z_score_validation_result, test_backends): if "postgresql" not in test_backends: pytest.skip( "test_database_store_backend_get_url_for_key requires postgresql") df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]}) validator = build_sa_validator_with_data(df=df, sa_engine_name="postgresql") result = validator.expect_column_value_z_scores_to_be_less_than( column="a", mostly=0.9, threshold=4, double_sided=True) assert result == z_score_validation_result
def test_expect_column_values_to_be_of_type_string_dialect_pyathena(sa): from pyathena import sqlalchemy_athena df = pd.DataFrame({"col": ["test_val1", "test_val2"]}) validator = build_sa_validator_with_data(df, "sqlite") # Monkey-patch dialect for testing purposes. validator.execution_engine.dialect_module = sqlalchemy_athena result = validator.expect_column_values_to_be_of_type("col", type_="STRINGTYPE") assert result == ExpectationValidationResult( success=True, expectation_config={ "expectation_type": "expect_column_values_to_be_of_type", "kwargs": { "column": "col", "type_": "STRINGTYPE", }, "meta": {}, }, result={ "element_count": 2, "unexpected_count": 0, "unexpected_percent": 0.0, "partial_unexpected_list": [], "missing_count": 0, "missing_percent": 0.0, "unexpected_percent_total": 0.0, "unexpected_percent_nonmissing": 0.0, }, exception_info={ "raised_exception": False, "exception_traceback": None, "exception_message": None, }, meta={}, )
def test_table_column_reflection_fallback(test_backends, sa): include_sqlalchemy: bool = "sqlite" in test_backends include_postgresql: bool = "postgresql" in test_backends include_mysql: bool = "mysql" in test_backends include_mssql: bool = "mssql" in test_backends include_bigquery: bool = "bigquery" in test_backends if not create_engine: pytest.skip("Unable to import sqlalchemy.create_engine() -- skipping.") test_backend_names: List[str] = build_test_backends_list_v3( include_pandas=False, include_spark=False, include_sqlalchemy=include_sqlalchemy, include_postgresql=include_postgresql, include_mysql=include_mysql, include_mssql=include_mssql, include_bigquery=include_bigquery, ) df: pd.DataFrame = pd.DataFrame({ "name": ["Frank", "Steve", "Jane", "Frank", "Michael"], "age": [16, 21, 38, 22, 10], "pet": ["fish", "python", "cat", "python", "frog"], }) validators_config: Dict[str, Validator] = {} validator: Validator backend_name: str table_name: str for backend_name in test_backend_names: if backend_name in ["sqlite", "postgresql", "mysql", "mssql"]: table_name = generate_test_table_name() validator = build_sa_validator_with_data( df=df, sa_engine_name=backend_name, schemas=None, caching=True, table_name=table_name, sqlite_db_path=None, ) if validator is not None: validators_config[table_name] = validator engine: Engine metrics: dict = {} table_columns_metric: MetricConfiguration results: dict reflected_columns_list: List[Dict[str, str]] reflected_column_config: Dict[str, str] column_name: str validation_result: ExpectationValidationResult for table_name, validator in validators_config.items(): table_columns_metric, results = get_table_columns_metric( engine=validator.execution_engine) metrics.update(results) assert set(metrics[table_columns_metric.id]) == {"name", "age", "pet"} selectable: Select = sqlalchemy.Table( table_name, sqlalchemy.MetaData(), schema=None, ) reflected_columns_list = column_reflection_fallback( selectable=selectable, dialect=validator.execution_engine.engine.dialect, sqlalchemy_engine=validator.execution_engine.engine, ) for column_name in [ reflected_column_config["name"] for reflected_column_config in reflected_columns_list ]: validation_result = validator.expect_column_to_exist( column=column_name) assert validation_result.success if validators_config: validator = list(validators_config.values())[0] validation_result = validator.expect_column_mean_to_be_between( column="age", min_value=10) assert validation_result.success validation_result = validator.expect_table_row_count_to_equal(value=5) assert validation_result.success validation_result = validator.expect_table_row_count_to_equal(value=3) assert not validation_result.success