def test_BasicDatasetProfiler_null_column(): """ The profiler should determine that null columns are of null cardinality and of null type and not to generate expectations specific to types and cardinality categories. We verify this by running the basic profiler on a Pandas dataset with an empty column and asserting the number of successful results for the empty columns. """ toy_dataset = PandasDataset({"x": [1, 2, 3], "y": [None, None, None]}) assert (len( toy_dataset.get_expectation_suite( suppress_warnings=True).expectations) == 0) expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset) # TODO: assert set - specific expectations assert (len([ result for result in evr_config["results"] if result.expectation_config["kwargs"].get("column") == "y" and result.success ]) == 4) assert len([ result for result in evr_config["results"] if result.expectation_config["kwargs"].get("column") == "y" and result.success ]) < len([ result for result in evr_config["results"] if result.expectation_config["kwargs"].get("column") == "x" and result.success ])
def test_BasicDatasetProfiler_with_context(filesystem_csv_data_context): context = filesystem_csv_data_context context.create_expectation_suite("default") datasource = context.datasources["rad_datasource"] base_dir = datasource.config["batch_kwargs_generators"]["subdir_reader"][ "base_directory"] batch_kwargs = { "datasource": "rad_datasource", "path": os.path.join(base_dir, "f1.csv"), } batch = context.get_batch(batch_kwargs, "default") expectation_suite, validation_results = BasicDatasetProfiler.profile(batch) assert expectation_suite.expectation_suite_name == "default" assert "BasicDatasetProfiler" in expectation_suite.meta assert set(expectation_suite.meta["BasicDatasetProfiler"].keys()) == { "created_by", "created_at", "batch_kwargs", } assert (expectation_suite.meta["BasicDatasetProfiler"]["batch_kwargs"] == batch_kwargs) for exp in expectation_suite.expectations: assert "BasicDatasetProfiler" in exp.meta assert "confidence" in exp.meta["BasicDatasetProfiler"] assert set(validation_results.meta.keys()) == { "batch_kwargs", "batch_markers", "batch_parameters", "expectation_suite_name", "great_expectations.__version__", "run_id", }
def test_BasicDatasetProfiler_non_numeric_low_cardinality( non_numeric_low_card_dataset): """ Unit test to check the expectations that BasicDatasetProfiler creates for a low cardinality non numeric column. The test is executed against all the backends (Pandas, Spark, etc.), because it uses the fixture. """ expectations_config, evr_config = BasicDatasetProfiler.profile( non_numeric_low_card_dataset) assert set([ "expect_column_to_exist", "expect_column_values_to_be_in_type_list", "expect_column_unique_value_count_to_be_between", 'expect_column_distinct_values_to_be_in_set', "expect_column_proportion_of_unique_values_to_be_between", "expect_column_values_to_not_be_null", "expect_column_values_to_be_in_set", "expect_column_values_to_not_match_regex", ]) == set([ expectation.expectation_type for expectation in expectations_config.expectations if expectation.kwargs.get("column") == "lowcardnonnum" ])
def test_BasicDatasetProfiler_numeric_high_cardinality( numeric_high_card_dataset): """ Unit test to check the expectations that BasicDatasetProfiler creates for a high cardinality numeric column. The test is executed against all the backends (Pandas, Spark, etc.), because it uses the fixture. """ expectations_config, evr_config = BasicDatasetProfiler.profile( numeric_high_card_dataset) assert set([ "expect_column_to_exist", "expect_table_row_count_to_be_between", "expect_table_columns_to_match_ordered_list", "expect_column_values_to_be_in_type_list", "expect_column_unique_value_count_to_be_between", "expect_column_proportion_of_unique_values_to_be_between", "expect_column_values_to_not_be_null", "expect_column_values_to_be_in_set", "expect_column_values_to_be_unique", ]) == set([ expectation.expectation_type for expectation in expectations_config.expectations ])
def test_BasicDatasetProfiler_null_column(): """ The profiler should determine that null columns are of null cardinality and of null type and not to generate expectations specific to types and cardinality categories. We verify this by running the basic profiler on a Pandas dataset with an empty column and asserting the number of successful results for the empty columns. """ toy_dataset = PandasDataset({ "x": [1, 2, 3], "y": [None, None, None] }, data_asset_name="toy_dataset") assert len( toy_dataset.get_expectation_suite( suppress_warnings=True)["expectations"]) == 0 expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset) # TODO: assert set - specific expectations assert len([ result for result in evr_config['results'] if result['expectation_config']['kwargs'].get('column') == 'y' and result['success'] ]) == 4 assert len([result for result in evr_config['results'] if result['expectation_config']['kwargs'].get('column') == 'y' and result['success']]) < \ len([result for result in evr_config['results'] if result['expectation_config']['kwargs'].get('column') == 'x' and result['success']])
def test_BasicDatasetProfiler_with_context(empty_data_context, filesystem_csv_2): empty_data_context.add_datasource("my_datasource", "pandas", base_directory=str(filesystem_csv_2)) not_so_empty_data_context = empty_data_context batch = not_so_empty_data_context.get_batch("my_datasource/f1") expectations_config, validation_results = BasicDatasetProfiler.profile( batch) # print(batch.get_batch_kwargs()) # print(json.dumps(expectations_config, indent=2)) assert expectations_config["data_asset_name"] == "my_datasource/default/f1" assert expectations_config["expectation_suite_name"] == "default" assert "BasicDatasetProfiler" in expectations_config["meta"] assert set(expectations_config["meta"]["BasicDatasetProfiler"].keys()) == { "created_by", "created_at", "batch_kwargs" } for exp in expectations_config["expectations"]: assert "BasicDatasetProfiler" in exp["meta"] assert "confidence" in exp["meta"]["BasicDatasetProfiler"] assert validation_results["meta"][ "data_asset_name"] == "my_datasource/default/f1" assert set(validation_results["meta"].keys()) == { "great_expectations.__version__", "data_asset_name", "expectation_suite_name", "run_id", "batch_kwargs" }
def test_BasicDatasetProfiler_partially_null_column(dataset): """ Unit test to check the expectations that BasicDatasetProfiler creates for a partially null column. The test is executed against all the backends (Pandas, Spark, etc.), because it uses the fixture. "nulls" is the partially null column in the fixture dataset """ expectations_config, evr_config = BasicDatasetProfiler.profile(dataset) assert set(["expect_column_to_exist", "expect_column_values_to_be_in_type_list", "expect_column_unique_value_count_to_be_between", "expect_column_proportion_of_unique_values_to_be_between", "expect_column_values_to_not_be_null", "expect_column_values_to_be_in_set", "expect_column_values_to_be_unique"]) == \ set([expectation['expectation_type'] for expectation in expectations_config["expectations"] if expectation["kwargs"].get("column") == "nulls"])
def test_BasicDatasetProfiler(mock_emit): toy_dataset = PandasDataset({"x": [1, 2, 3]}, ) assert (len( toy_dataset.get_expectation_suite( suppress_warnings=True).expectations) == 0) expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset) assert (len( toy_dataset.get_expectation_suite(suppress_warnings=True).expectations) > 0) assert "BasicDatasetProfiler" in expectations_config.meta assert set(expectations_config.meta["BasicDatasetProfiler"].keys()) == { "created_by", "created_at", "batch_kwargs", } assert "notes" in expectations_config.meta assert set( expectations_config.meta["notes"].keys()) == {"format", "content"} assert "To add additional notes" in expectations_config.meta["notes"][ "content"][0] added_expectations = set() for exp in expectations_config.expectations: added_expectations.add(exp.expectation_type) assert "BasicDatasetProfiler" in exp.meta assert "confidence" in exp.meta["BasicDatasetProfiler"] expected_expectations = { "expect_table_row_count_to_be_between", "expect_table_columns_to_match_ordered_list", "expect_column_values_to_be_in_set", "expect_column_unique_value_count_to_be_between", "expect_column_proportion_of_unique_values_to_be_between", "expect_column_values_to_not_be_null", "expect_column_values_to_be_in_type_list", "expect_column_values_to_be_unique", } assert expected_expectations.issubset(added_expectations) # Note 20211209 - Currently the only method called by the Profiler that is instrumented for usage_statistics # is ExpectationSuite's add_expectation(). It will not send a usage_stats event when called from a Profiler. # this number can change in the future our instrumentation changes. assert mock_emit.call_count == 0 assert mock_emit.call_args_list == []
def test_BasicDatasetProfiler(): toy_dataset = PandasDataset({"x": [1, 2, 3]}, data_asset_name="toy_dataset") assert len( toy_dataset.get_expectation_suite( suppress_warnings=True)["expectations"]) == 0 expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset) # print(json.dumps(expectations_config, indent=2)) assert len( toy_dataset.get_expectation_suite( suppress_warnings=True)["expectations"]) > 0 assert expectations_config["data_asset_name"] == "toy_dataset" assert "BasicDatasetProfiler" in expectations_config["meta"] assert set(expectations_config["meta"]["BasicDatasetProfiler"].keys()) == { "created_by", "created_at" } assert "notes" in expectations_config["meta"] assert set( expectations_config["meta"]["notes"].keys()) == {"format", "content"} assert "To add additional notes" in expectations_config["meta"]["notes"][ "content"][0] added_expectations = set() for exp in expectations_config["expectations"]: added_expectations.add(exp["expectation_type"]) assert "BasicDatasetProfiler" in exp["meta"] assert "confidence" in exp["meta"]["BasicDatasetProfiler"] expected_expectations = { 'expect_table_row_count_to_be_between', 'expect_table_columns_to_match_ordered_list', 'expect_column_values_to_be_in_set', 'expect_column_unique_value_count_to_be_between', 'expect_column_proportion_of_unique_values_to_be_between', 'expect_column_values_to_not_be_null', 'expect_column_values_to_be_in_type_list', 'expect_column_values_to_be_unique' } assert expected_expectations.issubset(added_expectations)
def test_BasicDatasetProfiler(): toy_dataset = PandasDataset({"x": [1, 2, 3]}, ) assert (len( toy_dataset.get_expectation_suite( suppress_warnings=True).expectations) == 0) expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset) assert (len( toy_dataset.get_expectation_suite(suppress_warnings=True).expectations) > 0) assert "BasicDatasetProfiler" in expectations_config.meta assert set(expectations_config.meta["BasicDatasetProfiler"].keys()) == { "created_by", "created_at", "batch_kwargs", } assert "notes" in expectations_config.meta assert set( expectations_config.meta["notes"].keys()) == {"format", "content"} assert "To add additional notes" in expectations_config.meta["notes"][ "content"][0] added_expectations = set() for exp in expectations_config.expectations: added_expectations.add(exp.expectation_type) assert "BasicDatasetProfiler" in exp.meta assert "confidence" in exp.meta["BasicDatasetProfiler"] expected_expectations = { "expect_table_row_count_to_be_between", "expect_table_columns_to_match_ordered_list", "expect_column_values_to_be_in_set", "expect_column_unique_value_count_to_be_between", "expect_column_proportion_of_unique_values_to_be_between", "expect_column_values_to_not_be_null", "expect_column_values_to_be_in_type_list", "expect_column_values_to_be_unique", } assert expected_expectations.issubset(added_expectations)