def _ge_context(self) -> Iterator[GEContext]: with self.base_engine.connect() as conn: data_context = BaseDataContext(project_config=DataContextConfig( # The datasource will be added via add_datasource(). datasources={}, store_backend_defaults=InMemoryStoreBackendDefaults(), anonymous_usage_statistics={ "enabled": False, # "data_context_id": <not set>, }, )) datasource_name = f"{self._datasource_name_base}-{uuid.uuid4()}" datasource_config = DatasourceConfig( class_name="SqlAlchemyDatasource", credentials={ # This isn't actually used since we pass the connection directly, # but GE parses it to change some of its behavior so it's useful # to emulate that here. "url": conn.engine.url, }, ) with _inject_connection_into_datasource(conn): # Using the add_datasource method ensures that the datasource is added to # GE-internal cache, which avoids problems when calling GE methods later on. assert data_context.add_datasource( datasource_name, initialize=True, **dict(datasourceConfigSchema.dump(datasource_config)), ) assert data_context.get_datasource(datasource_name) yield GEContext(data_context, datasource_name)
def __init__(self, conn, report): self.conn = conn self.report = report data_context_config = DataContextConfig( datasources={ self.datasource_name: DatasourceConfig( class_name="SqlAlchemyDatasource", credentials={ # This isn't actually used since we pass the connection directly, # but GE parses it to change some of its behavior so it's useful # to emulate that here. "url": self.conn.engine.url, }, ) }, store_backend_defaults=InMemoryStoreBackendDefaults(), anonymous_usage_statistics={ "enabled": False, # "data_context_id": <not set>, }, ) with _properly_init_datasource(self.conn): self.data_context = BaseDataContext( project_config=data_context_config)
def test_data_context_concurrency_property(): data_context = BaseDataContext( project_config=DataContextConfig( concurrency=ConcurrencyConfig(enabled=True), store_backend_defaults=InMemoryStoreBackendDefaults(), ) ) assert data_context.concurrency.enabled
def validate_with_great_expectations( self, dataframe: TypeVar("pyspark.sql.DataFrame"), # noqa: F821 expectation_suite: TypeVar("ge.core.ExpectationSuite"), # noqa: F821 ge_validate_kwargs: Optional[dict], ): # NOTE: InMemoryStoreBackendDefaults SHOULD NOT BE USED in normal settings. You # may experience data loss as it persists nothing. It is used here for testing. # Please refer to docs to learn how to instantiate your DataContext. store_backend_defaults = InMemoryStoreBackendDefaults() data_context_config = DataContextConfig( store_backend_defaults=store_backend_defaults, checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) context = BaseDataContext(project_config=data_context_config) datasource = { "name": "my_spark_dataframe", "class_name": "Datasource", "execution_engine": { "class_name": "SparkDFExecutionEngine", "force_reuse_spark_context": True, }, "data_connectors": { "default_runtime_data_connector_name": { "class_name": "RuntimeDataConnector", "batch_identifiers": ["batch_id"], } }, } context.add_datasource(**datasource) # Here is a RuntimeBatchRequest using a dataframe batch_request = RuntimeBatchRequest( datasource_name="my_spark_dataframe", data_connector_name="default_runtime_data_connector_name", data_asset_name= "<YOUR_MEANGINGFUL_NAME>", # This can be anything that identifies this data_asset for you batch_identifiers={"batch_id": "default_identifier"}, runtime_parameters={"batch_data": dataframe}, # Your dataframe goes here ) context.save_expectation_suite(expectation_suite) validator = context.get_validator( batch_request=batch_request, expectation_suite_name=expectation_suite.expectation_suite_name, ) report = validator.validate(**ge_validate_kwargs) return report
def build_in_memory_runtime_context(): data_context_config: DataContextConfig = DataContextConfig( datasources={ "pandas_datasource": { "execution_engine": { "class_name": "PandasExecutionEngine", "module_name": "great_expectations.execution_engine", }, "class_name": "Datasource", "module_name": "great_expectations.datasource", "data_connectors": { "runtime_data_connector": { "class_name": "RuntimeDataConnector", "batch_identifiers": [ "id_key_0", "id_key_1", ], } }, }, "spark_datasource": { "execution_engine": { "class_name": "SparkDFExecutionEngine", "module_name": "great_expectations.execution_engine", }, "class_name": "Datasource", "module_name": "great_expectations.datasource", "data_connectors": { "runtime_data_connector": { "class_name": "RuntimeDataConnector", "batch_identifiers": [ "id_key_0", "id_key_1", ], } }, }, }, expectations_store_name="expectations_store", validations_store_name="validations_store", evaluation_parameter_store_name="evaluation_parameter_store", checkpoint_store_name="checkpoint_store", store_backend_defaults=InMemoryStoreBackendDefaults(), ) context: BaseDataContext = BaseDataContext( project_config=data_context_config) return context
from typing import List from ruamel import yaml import great_expectations as ge from great_expectations.core.batch import Batch, BatchRequest, RuntimeBatchRequest from great_expectations.data_context import BaseDataContext from great_expectations.data_context.types.base import ( DataContextConfig, InMemoryStoreBackendDefaults, ) # NOTE: InMemoryStoreBackendDefaults SHOULD NOT BE USED in normal settings. You # may experience data loss as it persists nothing. It is used here for testing. # Please refer to docs to learn how to instantiate your DataContext. store_backend_defaults = InMemoryStoreBackendDefaults() data_context_config = DataContextConfig( store_backend_defaults=store_backend_defaults, checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) context = BaseDataContext(project_config=data_context_config) datasource_config = { "name": "my_gcs_datasource", "class_name": "Datasource", "execution_engine": { "class_name": "SparkDFExecutionEngine" }, "data_connectors": { "default_runtime_data_connector_name": { "class_name": "RuntimeDataConnector",
def _create_context( backend_api: str, datasource_name: str, data_connector_name: str, asset_names: List[str], html_dir: Optional[str] = None, ) -> DataContext: data_docs_sites = ( { "local_site": { "class_name": "SiteBuilder", "show_how_to_buttons": False, "store_backend": { "class_name": "TupleFilesystemStoreBackend", "base_directory": html_dir, }, } } if html_dir else None ) bigquery_project = os.environ.get("GE_TEST_GCP_PROJECT") if not bigquery_project: raise ValueError( "Environment Variable GE_TEST_GCP_PROJECT is required to run BigQuery performance tests" ) bigquery_dataset = os.environ.get("GE_TEST_BIGQUERY_PERFORMANCE_DATASET") if not bigquery_dataset: raise ValueError( "Environment Variable GE_TEST_BIGQUERY_PERFORMANCE_DATASET is required to run BigQuery performance tests" ) data_context_config = DataContextConfig( store_backend_defaults=InMemoryStoreBackendDefaults(), data_docs_sites=data_docs_sites, anonymous_usage_statistics={"enabled": False}, concurrency=concurrency_config(), ) context = BaseDataContext(project_config=data_context_config) if backend_api == "V3": datasource_config = { "name": datasource_name, "class_name": "Datasource", "execution_engine": { "class_name": "SqlAlchemyExecutionEngine", "connection_string": f"bigquery://{bigquery_project}/{bigquery_dataset}", }, "data_connectors": { data_connector_name: { "class_name": "ConfiguredAssetSqlDataConnector", "name": "whole_table", "assets": {asset_name: {} for asset_name in asset_names}, }, }, } elif backend_api == "V2": datasource_config = { "name": datasource_name, "credentials": { "url": f"bigquery://{bigquery_project}/{bigquery_dataset}", }, "class_name": "SqlAlchemyDatasource", "module_name": "great_expectations.datasource", "batch_kwargs_generators": {}, "data_asset_type": { "module_name": "great_expectations.dataset", "class_name": "SqlAlchemyDataset", }, } else: raise ValueError(f"Unsupported backend_api {backend_api}") context.add_datasource(**datasource_config) return context