def test_dataset_from_pandas_source(tmpdir): data_file = tmpdir + '/data.json' json_data = [ {"name": "my name", "birthdate": "2020-10-01", "address": "1234 Main st", "size": 12}, {"name": "your name", "birthdate": "2020-06-01", "address": "1313 Mockingbird Ln", "size": 12} ] with open(data_file, mode='w') as out: json.dump(json_data, out) store_defaults = FilesystemStoreBackendDefaults(root_directory=tmpdir) project_config.stores = store_defaults.stores project_config.expectations_store_name = store_defaults.expectations_store_name project_config.validations_store_name = store_defaults.validations_store_name project_config.checkpoint_store_name = store_defaults.checkpoint_store_name ctx = BaseDataContext(project_config=project_config) pd_dataset = PandasDataset(pandas.read_json(data_file), **{'batch_kwargs': {'path': 'gcs://my_bucket/path/to/my/data'}, 'data_context': ctx}) action = OpenLineageValidationAction(ctx, openlineage_host='http://localhost:5000', openlineage_namespace='test_ns', job_name='test_job') datasets = action._fetch_datasets_from_pandas_source(pd_dataset, validation_result_suite=result_suite) assert len(datasets) == 1 input_ds = datasets[0] assert input_ds.name == '/path/to/my/data' assert input_ds.namespace == "gcs://my_bucket" assert "dataSource" in input_ds.facets assert input_ds.facets["dataSource"].name == "gcs://my_bucket" assert input_ds.facets["dataSource"].uri == 'gcs://my_bucket' assert 'schema' in input_ds.facets assert len(input_ds.facets['schema'].fields) == 4 assert all(f in input_ds.facets['schema'].fields for f in [SchemaField('name', 'object'), SchemaField('birthdate', 'object'), SchemaField('address', 'object'), SchemaField('size', 'int64')]) assert len(input_ds.inputFacets) == 3 assert all(k in input_ds.inputFacets for k in ['dataQuality', 'greatExpectations_assertions', 'dataQualityMetrics']) assert input_ds.inputFacets['dataQuality'].rowCount == 10 assert 'size' in input_ds.inputFacets['dataQuality'].columnMetrics assert input_ds.inputFacets['dataQuality'].columnMetrics['size'].sum == 60 assert len(input_ds.inputFacets['greatExpectations_assertions'].assertions) == 2 assert all(a in input_ds.inputFacets['greatExpectations_assertions'].assertions for a in [GreatExpectationsAssertion('expect_table_row_count_to_equal', True), GreatExpectationsAssertion('expect_column_sum_to_be_between', True, 'size')])
def test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaults( construct_data_context_config, default_pandas_datasource_config ): """ What does this test and why? Ensure that a very simple DataContextConfig setup using FilesystemStoreBackendDefaults is created accurately This test sets the root_dir parameter """ test_root_directory = "test_root_dir" store_backend_defaults = FilesystemStoreBackendDefaults( root_directory=test_root_directory ) data_context_config = DataContextConfig( datasources={ "my_pandas_datasource": DatasourceConfig( class_name="PandasDatasource", batch_kwargs_generators={ "subdir_reader": { "class_name": "SubdirReaderBatchKwargsGenerator", "base_directory": "../data/", } }, ) }, store_backend_defaults=store_backend_defaults, ) # Create desired config data_context_id = data_context_config.anonymous_usage_statistics.data_context_id desired_config = construct_data_context_config( data_context_id=data_context_id, datasources=default_pandas_datasource_config ) # Add root_directory to stores and data_docs desired_config["stores"][desired_config["expectations_store_name"]][ "store_backend" ]["root_directory"] = test_root_directory desired_config["stores"][desired_config["validations_store_name"]]["store_backend"][ "root_directory" ] = test_root_directory desired_config["stores"][desired_config["checkpoint_store_name"]]["store_backend"][ "root_directory" ] = test_root_directory desired_config["data_docs_sites"]["local_site"]["store_backend"][ "root_directory" ] = test_root_directory data_context_config_schema = DataContextConfigSchema() assert filter_properties_dict( properties=data_context_config_schema.dump(data_context_config) ) == filter_properties_dict(properties=desired_config) assert DataContext.validate_config(project_config=data_context_config)
def build_context(self): """ Purpose: Create a dataContext and datasource and add to object Returns: saves dataContext and datasource to self """ self.context=ge.get_context() #create datasource configuration datasource_config = { "name": "example_datasource", "class_name": "Datasource", "module_name": "great_expectations.datasource", "execution_engine": { "module_name": "great_expectations.execution_engine", "class_name": "PandasExecutionEngine", }, "data_connectors": { "default_runtime_data_connector_name": { "class_name": "RuntimeDataConnector", "batch_identifiers": ["default_identifier_name"], }, }, } #create data context configuration data_context_config = DataContextConfig( datasources={ "pandas": DatasourceConfig( class_name="Datasource", execution_engine={ "class_name": "PandasExecutionEngine" }, data_connectors={ "default_runtime_data_connector_name": { "class_name": "RuntimeDataConnector", "batch_identifiers": ["default_identifier_name"], } }, ) }, store_backend_defaults=FilesystemStoreBackendDefaults(root_directory=os.path.join(os.getcwd(),'great_expectations')), ) #build context and add data source self.context=BaseDataContext(project_config=data_context_config) #self.context.test_yaml_config(yaml.dump(datasource_config)) self.context.add_datasource(**datasource_config)
def test_dataset_from_sql_source(test_db_file, tmpdir): connection_url = f'sqlite:///{test_db_file}' engine = create_engine(connection_url) ds = SqlAlchemyDataset(table_name=TABLE_NAME, engine=engine) store_defaults = FilesystemStoreBackendDefaults(root_directory=tmpdir) project_config.stores = store_defaults.stores project_config.expectations_store_name = store_defaults.expectations_store_name project_config.validations_store_name = store_defaults.validations_store_name project_config.checkpoint_store_name = store_defaults.checkpoint_store_name ctx = BaseDataContext(project_config=project_config) action = OpenLineageValidationAction(ctx, openlineage_host='http://localhost:5000', openlineage_namespace='test_ns', job_name='test_job') datasets = action._fetch_datasets_from_sql_source(ds, result_suite) assert datasets is not None assert len(datasets) == 1 input_ds = datasets[0] assert input_ds.name == TABLE_NAME assert input_ds.namespace == "sqlite" assert "dataSource" in input_ds.facets assert input_ds.facets["dataSource"].name == "sqlite" assert input_ds.facets["dataSource"].uri == "sqlite:/" + test_db_file assert 'schema' in input_ds.facets assert len(input_ds.facets['schema'].fields) == 4 assert all(f in input_ds.facets['schema'].fields for f in [SchemaField('name', 'TEXT'), SchemaField('birthdate', 'TEXT'), SchemaField('address', 'TEXT'), SchemaField('size', 'INTEGER')]) assert len(input_ds.inputFacets) == 3 assert all(k in input_ds.inputFacets for k in ['dataQuality', 'greatExpectations_assertions', 'dataQualityMetrics']) assert input_ds.inputFacets['dataQuality'].rowCount == 10 assert 'size' in input_ds.inputFacets['dataQuality'].columnMetrics assert input_ds.inputFacets['dataQuality'].columnMetrics['size'].sum == 60 assert len(input_ds.inputFacets['greatExpectations_assertions'].assertions) == 2 assert all(a in input_ds.inputFacets['greatExpectations_assertions'].assertions for a in [GreatExpectationsAssertion('expect_table_row_count_to_equal', True), GreatExpectationsAssertion('expect_column_sum_to_be_between', True, 'size')])
def _get_ge_context_local(ge_project_path: str) -> BaseDataContext: """ This is configured to work with an in-memory pandas DataFrame. This setup allows us to run validations before (perhaps unnecessarily) writing any data to disk, as well as at any other stage. Currently using local storage. Args: ge_project_path (str): The path to the Great Expectations project, eg. `/home/viadot/my_flow`. Expectation suites need to be placed inside the `expectations` folder, eg. `/home/viadot/my_flow/expectations/failure.json`. Returns: BaseDataContext: The GE context (ie. config) required to run the validations. """ data_context_config = DataContextConfig( datasources={ "pandas": DatasourceConfig( class_name="PandasDatasource", batch_kwargs_generators={}, # override the CSV default ) }, store_backend_defaults=FilesystemStoreBackendDefaults(ge_project_path), validation_operators={ "action_list_operator": { "class_name": "ActionListValidationOperator", "action_list": [ { "name": "store_validation_result", "action": {"class_name": "StoreValidationResultAction"}, }, { "name": "store_evaluation_params", "action": {"class_name": "StoreEvaluationParametersAction"}, }, { "name": "update_data_docs", "action": {"class_name": "UpdateDataDocsAction"}, }, ], } }, ) context = BaseDataContext(project_config=data_context_config) return context
# 1. Install Great Expectations # %pip install great-expectations # Imports # 2. Set up Great Expectations # In-memory DataContext using DBFS and FilesystemStoreBackendDefaults # CODE vvvvv vvvvv # This root directory is for use in Databricks root_directory = "/dbfs/great_expectations/" # For testing purposes only, we change the root_directory to an ephemeral location created by our test runner root_directory = os.path.join(os.getcwd(), "dbfs_temp_directory") data_context_config = DataContextConfig( store_backend_defaults=FilesystemStoreBackendDefaults( root_directory=root_directory), ) context = BaseDataContext(project_config=data_context_config) # CODE ^^^^^ ^^^^^ # ASSERTIONS vvvvv vvvvv # Check the stores were initialized uncommitted_directory = os.path.join(root_directory, "uncommitted") assert {"checkpoints", "expectations", "uncommitted"}.issubset(set(os.listdir(root_directory))) assert os.listdir(uncommitted_directory) == ["validations"] # ASSERTIONS ^^^^^ ^^^^^ # 3. Prepare your data # See guide
def test_dataset_from_custom_sql(test_db_file, tmpdir): connection_url = f'sqlite:///{test_db_file}' engine = create_engine(connection_url) engine.execute("""CREATE TABLE join_table (name text, workplace text, position text)""") custom_sql = f"""SELECT * FROM {TABLE_NAME} t INNER JOIN join_table j ON t.name=j.name""" # note the batch_kwarg key is 'query', but the constructor arg is 'custom_sql' ds = SqlAlchemyDataset(engine=engine, custom_sql=custom_sql, batch_kwargs={'query': custom_sql}) store_defaults = FilesystemStoreBackendDefaults(root_directory=tmpdir) project_config.stores = store_defaults.stores project_config.expectations_store_name = store_defaults.expectations_store_name project_config.validations_store_name = store_defaults.validations_store_name project_config.checkpoint_store_name = store_defaults.checkpoint_store_name ctx = BaseDataContext(project_config=project_config) action = OpenLineageValidationAction(ctx, openlineage_host='http://localhost:5000', openlineage_namespace='test_ns', job_name='test_job') datasets = action._fetch_datasets_from_sql_source(ds, result_suite) assert datasets is not None assert len(datasets) == 2 assert all(name in [TABLE_NAME, 'join_table'] for name in [ds.name for ds in datasets]) input_ds = next(ds for ds in datasets if ds.name == TABLE_NAME) assert "dataSource" in input_ds.facets assert input_ds.facets["dataSource"].name == "sqlite" assert input_ds.facets["dataSource"].uri == "sqlite:/" + test_db_file assert 'schema' in input_ds.facets assert len(input_ds.facets['schema'].fields) == 4 assert all(f in input_ds.facets['schema'].fields for f in [SchemaField('name', 'TEXT'), SchemaField('birthdate', 'TEXT'), SchemaField('address', 'TEXT'), SchemaField('size', 'INTEGER')]) assert len(input_ds.inputFacets) == 3 assert all(k in input_ds.inputFacets for k in ['dataQuality', 'greatExpectations_assertions', 'dataQualityMetrics']) assert input_ds.inputFacets['dataQuality'].rowCount == 10 assert 'size' in input_ds.inputFacets['dataQuality'].columnMetrics assert input_ds.inputFacets['dataQuality'].columnMetrics['size'].sum == 60 assert len(input_ds.inputFacets['greatExpectations_assertions'].assertions) == 2 assert all(a in input_ds.inputFacets['greatExpectations_assertions'].assertions for a in [GreatExpectationsAssertion('expect_table_row_count_to_equal', True), GreatExpectationsAssertion('expect_column_sum_to_be_between', True, 'size')]) input_ds = next(ds for ds in datasets if ds.name == 'join_table') assert 'schema' in input_ds.facets assert len(input_ds.facets['schema'].fields) == 3 assert all(f in input_ds.facets['schema'].fields for f in [SchemaField('name', 'TEXT'), SchemaField('workplace', 'TEXT'), SchemaField('position', 'TEXT')]) assert len(input_ds.inputFacets) == 3 assert all(k in input_ds.inputFacets for k in ['dataQuality', 'greatExpectations_assertions', 'dataQualityMetrics'])
GE was installed inside a zip file -which is a location allowed by PEP 273-). Therefore, this test is intended to be run after installing GE inside a zip file and then setting the appropriate PYTHONPATH env variable. If desired, this test can also be run after installing GE in a normal filesystem location (i.e. a directory). This test is OK if it finishes without raising an exception. To make it easier to debug this test, it prints: * The location of the GE library: to verify that we are testing the library that we want * The version of the GE library: idem * data_docs url: If everything works, this will be a url (e.g. starting with file://...) Additional info: https://github.com/great-expectations/great_expectations/issues/3772 and https://www.python.org/dev/peps/pep-0273/ """ print(f"Great Expectations location: {ge.__file__}") print(f"Great Expectations version: {ge.__version__}") data_context_config = DataContextConfig( datasources={ "example_datasource": DatasourceConfig(class_name="PandasDatasource") }, store_backend_defaults=FilesystemStoreBackendDefaults( root_directory=tempfile.mkdtemp() + os.sep + "my_greatexp_workdir"), ) context = BaseDataContext(project_config=data_context_config) print(f"Great Expectations data_docs url: {context.build_data_docs()}")