def test_expect_column_values_to_be_of_type_string_dialect_sqlite(sa):
df = pd.DataFrame({"col": ["test_val1", "test_val2"]})
validator = build_sa_validator_with_data(df, "sqlite")
result = validator.expect_column_values_to_be_of_type("col", type_="TEXT")
assert result == ExpectationValidationResult(
success=True,
expectation_config={
"expectation_type": "expect_column_values_to_be_of_type",
"kwargs": {
"column": "col",
"type_": "TEXT",
},
"meta": {},
},
result={
"element_count": 2,
"unexpected_count": 0,
"unexpected_percent": 0.0,
"partial_unexpected_list": [],
"missing_count": 0,
"missing_percent": 0.0,
"unexpected_percent_total": 0.0,
"unexpected_percent_nonmissing": 0.0,
},
exception_info={
"raised_exception": False,
"exception_traceback": None,
"exception_message": None,
},
meta={},
)
def test_sa_expect_column_value_z_scores_to_be_less_than_impl(
z_score_validation_result, test_backends):
if "postgresql" not in test_backends:
pytest.skip(
"test_database_store_backend_get_url_for_key requires postgresql")
df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
validator = build_sa_validator_with_data(df=df,
sa_engine_name="postgresql")
result = validator.expect_column_value_z_scores_to_be_less_than(
column="a", mostly=0.9, threshold=4, double_sided=True)
assert result == z_score_validation_result
def test_expect_column_values_to_be_of_type_string_dialect_pyathena(sa):
from pyathena import sqlalchemy_athena
df = pd.DataFrame({"col": ["test_val1", "test_val2"]})
validator = build_sa_validator_with_data(df, "sqlite")
# Monkey-patch dialect for testing purposes.
validator.execution_engine.dialect_module = sqlalchemy_athena
result = validator.expect_column_values_to_be_of_type("col",
type_="STRINGTYPE")
assert result == ExpectationValidationResult(
success=True,
expectation_config={
"expectation_type": "expect_column_values_to_be_of_type",
"kwargs": {
"column": "col",
"type_": "STRINGTYPE",
},
"meta": {},
},
result={
"element_count": 2,
"unexpected_count": 0,
"unexpected_percent": 0.0,
"partial_unexpected_list": [],
"missing_count": 0,
"missing_percent": 0.0,
"unexpected_percent_total": 0.0,
"unexpected_percent_nonmissing": 0.0,
},
exception_info={
"raised_exception": False,
"exception_traceback": None,
"exception_message": None,
},
meta={},
)
def test_table_column_reflection_fallback(test_backends, sa):
include_sqlalchemy: bool = "sqlite" in test_backends
include_postgresql: bool = "postgresql" in test_backends
include_mysql: bool = "mysql" in test_backends
include_mssql: bool = "mssql" in test_backends
include_bigquery: bool = "bigquery" in test_backends
if not create_engine:
pytest.skip("Unable to import sqlalchemy.create_engine() -- skipping.")
test_backend_names: List[str] = build_test_backends_list_v3(
include_pandas=False,
include_spark=False,
include_sqlalchemy=include_sqlalchemy,
include_postgresql=include_postgresql,
include_mysql=include_mysql,
include_mssql=include_mssql,
include_bigquery=include_bigquery,
)
df: pd.DataFrame = pd.DataFrame({
"name": ["Frank", "Steve", "Jane", "Frank", "Michael"],
"age": [16, 21, 38, 22, 10],
"pet": ["fish", "python", "cat", "python", "frog"],
})
validators_config: Dict[str, Validator] = {}
validator: Validator
backend_name: str
table_name: str
for backend_name in test_backend_names:
if backend_name in ["sqlite", "postgresql", "mysql", "mssql"]:
table_name = generate_test_table_name()
validator = build_sa_validator_with_data(
df=df,
sa_engine_name=backend_name,
schemas=None,
caching=True,
table_name=table_name,
sqlite_db_path=None,
)
if validator is not None:
validators_config[table_name] = validator
engine: Engine
metrics: dict = {}
table_columns_metric: MetricConfiguration
results: dict
reflected_columns_list: List[Dict[str, str]]
reflected_column_config: Dict[str, str]
column_name: str
validation_result: ExpectationValidationResult
for table_name, validator in validators_config.items():
table_columns_metric, results = get_table_columns_metric(
engine=validator.execution_engine)
metrics.update(results)
assert set(metrics[table_columns_metric.id]) == {"name", "age", "pet"}
selectable: Select = sqlalchemy.Table(
table_name,
sqlalchemy.MetaData(),
schema=None,
)
reflected_columns_list = column_reflection_fallback(
selectable=selectable,
dialect=validator.execution_engine.engine.dialect,
sqlalchemy_engine=validator.execution_engine.engine,
)
for column_name in [
reflected_column_config["name"]
for reflected_column_config in reflected_columns_list
]:
validation_result = validator.expect_column_to_exist(
column=column_name)
assert validation_result.success
if validators_config:
validator = list(validators_config.values())[0]
validation_result = validator.expect_column_mean_to_be_between(
column="age", min_value=10)
assert validation_result.success
validation_result = validator.expect_table_row_count_to_equal(value=5)
assert validation_result.success
validation_result = validator.expect_table_row_count_to_equal(value=3)
assert not validation_result.success