from typing import Dict, List
import pandas as pd
import pytest
import great_expectations
import great_expectations.exceptions as ge_exceptions
from great_expectations import DataContext
from great_expectations.core.batch import Batch, RuntimeBatchRequest
from great_expectations.core.id_dict import BatchSpec
from great_expectations.core.yaml_handler import YAMLHandler
from great_expectations.execution_engine.sparkdf_batch_data import SparkDFBatchData
from great_expectations.validator.validator import Validator
yaml = YAMLHandler()
@pytest.fixture()
def test_df_spark(spark_session):
test_df: "pyspark.sql.dataframe.DataFrame" = spark_session.createDataFrame(
data=pd.DataFrame(data={
"col1": [1, 2],
"col2": [3, 4]
}))
return test_df
@pytest.fixture
def data_context_with_datasource_spark_engine_batch_spec_passthrough(
empty_data_context, spark_session):
context: DataContext = empty_data_context
def yaml_handler() -> YAMLHandler:
return YAMLHandler()
from great_expectations.execution_engine.sqlalchemy_batch_data import (
SqlAlchemyBatchData, )
from tests.integration.fixtures.split_and_sample_data.splitter_test_cases_and_fixtures import (
TaxiSplittingTestCase,
TaxiSplittingTestCases,
TaxiTestData,
)
from tests.test_utils import (
LoadedTable,
clean_up_tables_with_prefix,
get_bigquery_connection_url,
get_snowflake_connection_url,
load_data_into_test_database,
)
yaml_handler: YAMLHandler = YAMLHandler()
def _get_connection_string_and_dialect() -> Tuple[str, str]:
with open("./connection_string.yml") as f:
db_config: dict = yaml_handler.load(f)
dialect: str = db_config["dialect"]
if dialect == "snowflake":
connection_string: str = get_snowflake_connection_url()
elif dialect == "bigquery":
connection_string: str = get_bigquery_connection_url()
else:
connection_string: str = db_config["connection_string"]
import os
from typing import List
import pytest
from great_expectations.core.yaml_handler import YAMLHandler
from great_expectations.data_context import BaseDataContext
from great_expectations.data_context.types.base import DataContextConfig
from great_expectations.data_context.store import ( # isort:skip
ExpectationsStore, ValidationsStore, EvaluationParameterStore,
)
yaml: YAMLHandler = YAMLHandler()
@pytest.fixture()
def basic_in_memory_data_context_config_just_stores():
return DataContextConfig(
config_version=3.0,
plugins_directory=None,
evaluation_parameter_store_name="evaluation_parameter_store",
expectations_store_name="expectations_store",
datasources={},
stores={
"expectations_store": {
"class_name": "ExpectationsStore"
},
"evaluation_parameter_store": {
"class_name": "EvaluationParameterStore"
},