def test_build_suite_no_config(titanic_validator, possible_expectations_set): """ What does this test do and why? Tests that the build_suite function works as expected with no config """ profiler = UserConfigurableProfiler(titanic_validator) suite = profiler.build_suite() expectations_from_suite = {i.expectation_type for i in suite.expectations} assert expectations_from_suite.issubset(possible_expectations_set) assert len(suite.expectations) == 48
def test_build_suite_with_semantic_types_dict( mock_emit, cardinality_dataset, possible_expectations_set, ): """ What does this test do and why? Tests that the build_suite function works as expected with a semantic_types dict """ semantic_types = { "numeric": ["col_few", "col_many", "col_very_many"], "value_set": ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_dataset, semantic_types_dict=semantic_types, primary_or_compound_key=["col_unique"], ignored_columns=["col_one"], value_set_threshold="unique", table_expectations_only=False, excluded_expectations=["expect_column_values_to_not_be_null"], ) suite = profiler.build_suite() ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) assert "column_one" not in columns_with_expectations assert "expect_column_values_to_not_be_null" not in expectations_from_suite assert expectations_from_suite.issubset(possible_expectations_set) assert len(suite.expectations) == 33 value_set_expectations = [ i for i in suite.expectations if i.expectation_type == "expect_column_values_to_be_in_set" ] value_set_columns = { i.kwargs.get("column") for i in value_set_expectations } assert len(value_set_columns) == 2 assert value_set_columns == {"col_two", "col_very_few"} # Note 20211209 - Currently the only method called by the Profiler that is instrumented for usage_statistics # is ExpectationSuite's add_expectation(). It will not send a usage_stats event when called from a Profiler. # this number can change in the future if our instrumentation changes. assert mock_emit.call_count == 0 assert mock_emit.call_args_list == []
def test__validate_config(cardinality_dataset): """ What does this test do and why? Tests the validate config function on the profiler """ with pytest.raises(AssertionError) as e: UserConfigurableProfiler(cardinality_dataset, ignored_columns="col_name") assert e.typename == "AssertionError" with pytest.raises(AssertionError) as e: UserConfigurableProfiler(cardinality_dataset, table_expectations_only="True") assert e.typename == "AssertionError"
def test_nullity_expectations_mostly_tolerance( nulls_dataset, possible_expectations_set ): excluded_expectations = [i for i in possible_expectations_set if "null" not in i] batch_df = nulls_dataset profiler = UserConfigurableProfiler( batch_df, excluded_expectations, not_null_only=False ) suite = profiler.build_suite() for i in suite.expectations: assert i["kwargs"]["mostly"] == 0.66
def test_build_suite_no_config( mock_emit, titanic_validator, possible_expectations_set, ): """ What does this test do and why? Tests that the build_suite function works as expected with no config """ profiler = UserConfigurableProfiler(titanic_validator) suite = profiler.build_suite() expectations_from_suite = {i.expectation_type for i in suite.expectations} assert expectations_from_suite.issubset(possible_expectations_set) assert len(suite.expectations) == 48 # Note 20211209 - Profiler will also call ExpectationSuite's add_expectation(), but it will not # send a usage_stats event when called from a Profiler. assert mock_emit.call_count == 1 assert "expectation_suite.add_expectation" not in [ mock_emit.call_args_list[0][0][0]["event"] ] # noinspection PyUnresolvedReferences expected_events: List[unittest.mock._Call] # noinspection PyUnresolvedReferences actual_events: List[unittest.mock._Call] expected_events = [ mock.call( { "event": "legacy_profiler.build_suite", "event_payload": { "profile_dataset_type": "Validator", "excluded_expectations_specified": False, "ignored_columns_specified": False, "not_null_only": False, "primary_or_compound_key_specified": False, "semantic_types_dict_specified": False, "table_expectations_only": False, "value_set_threshold_specified": True, "api_version": "v2", }, "success": True, } ), ] actual_events = mock_emit.call_args_list assert actual_events == expected_events
def test_profiler_init_full_config_no_semantic_types(cardinality_validator): """ What does this test do and why? Confirms that profiler initializes properly with a full config, without a semantic_types dict """ profiler = UserConfigurableProfiler( cardinality_validator, primary_or_compound_key=["col_unique"], ignored_columns=["col_one"], value_set_threshold="UNIQUE", table_expectations_only=False, excluded_expectations=["expect_column_values_to_not_be_null"], ) assert profiler.primary_or_compound_key == ["col_unique"] assert profiler.ignored_columns == [ "col_one", ] assert profiler.value_set_threshold == "UNIQUE" assert not profiler.table_expectations_only assert profiler.excluded_expectations == [ "expect_column_values_to_not_be_null" ] assert "col_one" not in profiler.column_info
def test_profiled_dataset_passes_own_validation(cardinality_validator, titanic_data_context): """ What does this test do and why? Confirms that a suite created on a validator with no config will pass when validated against itself """ context = titanic_data_context profiler = UserConfigurableProfiler(cardinality_validator, ignored_columns=["col_none"]) suite = profiler.build_suite() context.save_expectation_suite(suite) results = context.run_validation_operator( "action_list_operator", assets_to_validate=[cardinality_validator]) assert results["success"]
def test_user_configurable_profiler_progress_bar_config_enabled( mock_tqdm, cardinality_validator): semantic_types = { "numeric": ["col_few", "col_many", "col_very_many"], "value_set": ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_validator, semantic_types_dict=semantic_types, ) profiler.build_suite() assert mock_tqdm.called assert mock_tqdm.call_count == 1
def test_column_cardinality_functions(cardinality_validator): profiler = UserConfigurableProfiler(cardinality_validator) # assert profiler.column_info.get("col_none").get("cardinality") == "NONE" assert profiler.column_info.get("col_one").get("cardinality") == "ONE" assert profiler.column_info.get("col_two").get("cardinality") == "TWO" assert profiler.column_info.get("col_very_few").get( "cardinality") == "VERY_FEW" assert profiler.column_info.get("col_few").get("cardinality") == "FEW" assert profiler.column_info.get("col_many").get("cardinality") == "MANY" assert profiler.column_info.get("col_very_many").get( "cardinality") == "VERY_MANY" cardinality_with_ten_num_and_no_pct = ( OrderedProfilerCardinality.get_basic_column_cardinality(num_unique=10)) assert cardinality_with_ten_num_and_no_pct.name == "VERY_FEW" cardinality_with_unique_pct_and_no_num = ( OrderedProfilerCardinality.get_basic_column_cardinality( pct_unique=1.0)) assert cardinality_with_unique_pct_and_no_num.name == "UNIQUE" cardinality_with_no_pct_and_no_num = ( OrderedProfilerCardinality.get_basic_column_cardinality()) assert cardinality_with_no_pct_and_no_num.name == "NONE" cardinality_with_large_pct_and_no_num = ( OrderedProfilerCardinality.get_basic_column_cardinality( pct_unique=0.5)) assert cardinality_with_large_pct_and_no_num.name == "NONE"
def test_init_with_semantic_types(cardinality_dataset): """ What does this test do and why? Confirms that profiler initializes properly with a full config and a semantic_types dict """ semantic_types = { ProfilerSemanticTypes.NUMERIC.value: ["col_few", "col_many", "col_very_many"], ProfilerSemanticTypes.VALUE_SET.value: ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_dataset, semantic_types_dict=semantic_types, primary_or_compound_key=["col_unique"], ignored_columns=["col_one"], value_set_threshold="unique", table_expectations_only=False, excluded_expectations=["expect_column_values_to_not_be_null"], ) assert "col_one" not in profiler.column_info assert profiler.column_info.get("col_none") == { "cardinality": "NONE", "type": "NUMERIC", "semantic_types": [], } assert profiler.column_info.get("col_two") == { "cardinality": "TWO", "type": "INT", "semantic_types": ["VALUE_SET"], } assert profiler.column_info.get("col_very_few") == { "cardinality": "VERY_FEW", "type": "INT", "semantic_types": ["VALUE_SET"], } assert profiler.column_info.get("col_few") == { "cardinality": "FEW", "type": "INT", "semantic_types": ["NUMERIC"], } assert profiler.column_info.get("col_many") == { "cardinality": "MANY", "type": "INT", "semantic_types": ["NUMERIC"], } assert profiler.column_info.get("col_very_many") == { "cardinality": "VERY_MANY", "type": "INT", "semantic_types": ["NUMERIC"], } assert profiler.column_info.get("col_unique") == { "cardinality": "UNIQUE", "type": "INT", "semantic_types": [], }
def test_build_suite_with_semantic_types_dict( cardinality_validator, possible_expectations_set, ): """ What does this test do and why? Tests that the build_suite function works as expected with a semantic_types dict """ semantic_types = { "numeric": ["col_few", "col_many", "col_very_many"], "value_set": ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_validator, semantic_types_dict=semantic_types, primary_or_compound_key=["col_unique"], ignored_columns=["col_one"], value_set_threshold="unique", table_expectations_only=False, excluded_expectations=["expect_column_values_to_not_be_null"], ) suite = profiler.build_suite() ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) assert "column_one" not in columns_with_expectations assert "expect_column_values_to_not_be_null" not in expectations_from_suite assert expectations_from_suite.issubset(possible_expectations_set) assert len(suite.expectations) == 32 value_set_expectations = [ i for i in suite.expectations if i.expectation_type == "expect_column_values_to_be_in_set" ] value_set_columns = { i.kwargs.get("column") for i in value_set_expectations } assert len(value_set_columns) == 2 assert value_set_columns == {"col_two", "col_very_few"}
def test_profiler_init_no_config(cardinality_validator, ): """ What does this test do and why? Confirms that profiler can initialize with no config. """ profiler = UserConfigurableProfiler(cardinality_validator) assert profiler.primary_or_compound_key == [] assert profiler.ignored_columns == [] assert profiler.value_set_threshold == "MANY" assert not profiler.table_expectations_only assert profiler.excluded_expectations == []
def test_user_configurable_profiler_progress_bar_config_disabled( mock_tqdm, cardinality_validator): data_context = cardinality_validator.data_context data_context.project_config_with_variables_substituted.progress_bars = ( ProgressBarsConfig(profilers=False)) semantic_types = { "numeric": ["col_few", "col_many", "col_very_many"], "value_set": ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_validator, semantic_types_dict=semantic_types, ) profiler.build_suite() assert not mock_tqdm.called assert mock_tqdm.call_count == 0
def test_build_suite_when_suite_already_exists(cardinality_dataset): """ What does this test do and why? Confirms that creating a new suite on an existing profiler wipes the previous suite """ profiler = UserConfigurableProfiler( cardinality_dataset, table_expectations_only=True, excluded_expectations=["expect_table_row_count_to_be_between"], ) suite = profiler.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite(suite) assert len(suite.expectations) == 1 assert "expect_table_columns_to_match_ordered_list" in expectations profiler.excluded_expectations = ["expect_table_columns_to_match_ordered_list"] suite = profiler.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite(suite) assert len(suite.expectations) == 1 assert "expect_table_row_count_to_be_between" in expectations
def configurable_profiler(dataset: PandasDataset) -> ExpectationSuite: from great_expectations.profile.user_configurable_profiler import ( UserConfigurableProfiler, ) return UserConfigurableProfiler( profile_dataset=dataset, excluded_expectations=[ "expect_table_columns_to_match_ordered_list", "expect_table_row_count_to_be_between", ], value_set_threshold="few", ).build_suite()
def test__validate_semantic_types_dict(cardinality_validator): """ What does this test do and why? Tests that _validate_semantic_types_dict function errors when not formatted correctly """ bad_semantic_types_dict_type = {"value_set": "col_few"} with pytest.raises(AssertionError) as e: # noinspection PyTypeChecker UserConfigurableProfiler( cardinality_validator, semantic_types_dict=bad_semantic_types_dict_type ) assert e.value.args[0] == ( "Entries in semantic type dict must be lists of column names e.g. " "{'semantic_types': {'numeric': ['number_of_transactions']}}" ) bad_semantic_types_incorrect_type = {"incorrect_type": ["col_few"]} with pytest.raises(ValueError) as e: UserConfigurableProfiler( cardinality_validator, semantic_types_dict=bad_semantic_types_incorrect_type ) assert e.value.args[0] == ( f"incorrect_type is not a recognized semantic_type. Please only include one of " f"{[semantic_type.value for semantic_type in ProfilerSemanticTypes]}" ) # Error if column is specified for both semantic_types and ignored working_semantic_type = {"numeric": ["col_few"]} with pytest.raises(ValueError) as e: UserConfigurableProfiler( cardinality_validator, semantic_types_dict=working_semantic_type, ignored_columns=["col_few"], ) assert e.value.args[0] == ( "Column col_few is specified in both the semantic_types_dict and the list of ignored columns. Please remove " f"one of these entries to proceed." )
def test_config_with_not_null_only( titanic_data_context_modular_api, possible_expectations_set ): """ What does this test do and why? Confirms that the not_null_only key in config works as expected. """ excluded_expectations = [i for i in possible_expectations_set if "null" not in i] df = pd.DataFrame( { "mostly_null": [i if i % 3 == 0 else None for i in range(0, 1000)], "mostly_not_null": [None if i % 3 == 0 else i for i in range(0, 1000)], } ) validator = get_pandas_runtime_validator(titanic_data_context_modular_api, df) profiler_without_not_null_only = UserConfigurableProfiler( validator, excluded_expectations, not_null_only=False ) suite_without_not_null_only = profiler_without_not_null_only.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite( suite_without_not_null_only ) assert expectations == { "expect_column_values_to_be_null", "expect_column_values_to_not_be_null", } profiler_with_not_null_only = UserConfigurableProfiler( validator, excluded_expectations, not_null_only=True ) not_null_only_suite = profiler_with_not_null_only.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite( not_null_only_suite ) assert expectations == {"expect_column_values_to_not_be_null"} no_config_profiler = UserConfigurableProfiler(validator) no_config_suite = no_config_profiler.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite(no_config_suite) assert "expect_column_values_to_be_null" in expectations
def test_primary_or_compound_key_not_found_in_columns(mock_emit, cardinality_dataset): """ What does this test do and why? Confirms that an error is raised if a primary_or_compound key is specified with a column not found in the dataset """ # regular case, should pass working_profiler = UserConfigurableProfiler( cardinality_dataset, primary_or_compound_key=["col_unique"]) assert working_profiler.primary_or_compound_key == ["col_unique"] # key includes a non-existent column, should fail with pytest.raises(ValueError) as e: # noinspection PyUnusedLocal bad_key_profiler = UserConfigurableProfiler( cardinality_dataset, primary_or_compound_key=["col_unique", "col_that_does_not_exist"], ) assert e.value.args[0] == ( """Column col_that_does_not_exist not found. Please ensure that this column is in the PandasDataset if you \ would like to use it as a primary_or_compound_key. """) # key includes a column that exists, but is in ignored_columns, should pass ignored_column_profiler = UserConfigurableProfiler( cardinality_dataset, primary_or_compound_key=["col_unique", "col_one"], ignored_columns=["col_none", "col_one"], ) assert ignored_column_profiler.primary_or_compound_key == [ "col_unique", "col_one" ] # Note 20211209 - Currently the only method called by the Profiler that is instrumented for usage_statistics # is ExpectationSuite's add_expectation(). It will not send a usage_stats event when called from a Profiler. # this number can change in the future if our instrumentation changes. assert mock_emit.call_count == 0 assert mock_emit.call_args_list == []
def test_profiler_works_with_batch_object(cardinality_validator): profiler = UserConfigurableProfiler(cardinality_validator.active_batch) assert profiler.primary_or_compound_key == [] assert profiler.ignored_columns == [] assert profiler.value_set_threshold == "MANY" assert not profiler.table_expectations_only assert profiler.excluded_expectations == [] assert profiler.all_table_columns == [ "col_one", "col_two", "col_very_few", "col_few", "col_many", "col_very_many", "col_unique", ]
def test_all_table_columns_populates(taxi_validator_pandas): taxi_profiler = UserConfigurableProfiler(taxi_validator_pandas) assert taxi_profiler.all_table_columns == [ "vendor_id", "pickup_datetime", "dropoff_datetime", "passenger_count", "trip_distance", "rate_code_id", "store_and_fwd_flag", "pickup_location_id", "dropoff_location_id", "payment_type", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "improvement_surcharge", "total_amount", "congestion_surcharge", ]
def test_build_suite_with_semantic_types_dict( mock_emit, cardinality_validator, possible_expectations_set, ): """ What does this test do and why? Tests that the build_suite function works as expected with a semantic_types dict """ semantic_types = { "numeric": ["col_few", "col_many", "col_very_many"], "value_set": ["col_two", "col_very_few"], } profiler = UserConfigurableProfiler( cardinality_validator, semantic_types_dict=semantic_types, primary_or_compound_key=["col_unique"], ignored_columns=["col_one"], value_set_threshold="unique", table_expectations_only=False, excluded_expectations=["expect_column_values_to_not_be_null"], ) suite = profiler.build_suite() ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) assert "column_one" not in columns_with_expectations assert "expect_column_values_to_not_be_null" not in expectations_from_suite assert expectations_from_suite.issubset(possible_expectations_set) assert len(suite.expectations) == 32 value_set_expectations = [ i for i in suite.expectations if i.expectation_type == "expect_column_values_to_be_in_set" ] value_set_columns = {i.kwargs.get("column") for i in value_set_expectations} assert len(value_set_columns) == 2 assert value_set_columns == {"col_two", "col_very_few"} # Note 20211209 - Profiler will also call ExpectationSuite's add_expectation(), but it will not # send a usage_stats event when called from a Profiler. assert mock_emit.call_count == 1 # noinspection PyUnresolvedReferences expected_events: List[unittest.mock._Call] # noinspection PyUnresolvedReferences actual_events: List[unittest.mock._Call] expected_events = [ mock.call( { "event": "legacy_profiler.build_suite", "event_payload": { "profile_dataset_type": "Validator", "excluded_expectations_specified": True, "ignored_columns_specified": True, "not_null_only": False, "primary_or_compound_key_specified": True, "semantic_types_dict_specified": True, "table_expectations_only": False, "value_set_threshold_specified": True, "api_version": "v2", }, "success": True, } ), ] actual_events = mock_emit.call_args_list assert actual_events == expected_events
def test_profiler_all_expectation_types_sqlalchemy( titanic_data_context_modular_api, taxi_validator_sqlalchemy, possible_expectations_set, ): """ What does this test do and why? Ensures that all available expectation types work as expected for sqlalchemy """ if taxi_validator_sqlalchemy == None: pytest.skip("a message") context = titanic_data_context_modular_api ignored_columns = [ "pickup_location_id", "dropoff_location_id", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "improvement_surcharge", "congestion_surcharge", ] semantic_types = { "datetime": ["pickup_datetime", "dropoff_datetime"], "numeric": ["total_amount", "passenger_count"], "value_set": [ "payment_type", "rate_code_id", "store_and_fwd_flag", "passenger_count", ], "boolean": ["store_and_fwd_flag"], } profiler = UserConfigurableProfiler( taxi_validator_sqlalchemy, semantic_types_dict=semantic_types, ignored_columns=ignored_columns, # TODO: Add primary_or_compound_key test # primary_or_compound_key=[ # "vendor_id", # "pickup_datetime", # "dropoff_datetime", # "trip_distance", # "pickup_location_id", # "dropoff_location_id", # ], ) assert profiler.column_info.get("rate_code_id") suite = profiler.build_suite() assert len(suite.expectations) == 45 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) unexpected_expectations = { "expect_column_values_to_be_unique", "expect_column_values_to_be_null", "expect_compound_columns_to_be_unique", } assert expectations_from_suite == { i for i in possible_expectations_set if i not in unexpected_expectations } ignored_included_columns_overlap = [ i for i in columns_with_expectations if i in ignored_columns ] assert len(ignored_included_columns_overlap) == 0 results = context.run_validation_operator( "action_list_operator", assets_to_validate=[taxi_validator_sqlalchemy] ) assert results["success"]
def test_expect_compound_columns_to_be_unique( taxi_validator_spark, taxi_data_ignored_columns, caplog ): """ Until all ExecutionEngine implementations for V3 are completed for this expectation: 1) Use the "taxi_validator_" argument for this test method, corresponding to one of the ExecutionEngine subclasses, for which this expectation has not yet been implemented (and update the :param annotation below accordingly); 2) With every additional ExecutionEngine implementation for this expectation, update the corresponding "test_profiler_all_expectation_types_" test method to include this expectation in the appropriate assertion. 3) Once this expectation has been implemented for all ExecutionEngine subclasses, delete this test method entirely. :param taxi_validator_spark: :param taxi_data_ignored_columns: :param caplog: :return: """ taxi_validator = taxi_validator_spark ignored_columns = taxi_data_ignored_columns + [ "pickup_datetime", "dropoff_datetime", "total_amount", "passenger_count", "payment_type", "rate_code_id", "store_and_fwd_flag", "passenger_count", "store_and_fwd_flag", "vendor_id", "trip_distance", ] profiler = UserConfigurableProfiler( taxi_validator, ignored_columns=ignored_columns, primary_or_compound_key=[ "vendor_id", "pickup_datetime", "dropoff_datetime", "trip_distance", "pickup_location_id", "dropoff_location_id", ], ) with caplog.at_level(logging.WARNING): suite = profiler.build_suite() log_warning_records = list( filter(lambda record: record.levelname == "WARNING", caplog.records) ) assert len(log_warning_records) == 0 assert len(suite.expectations) == 3 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) expected_expectations = { "expect_table_columns_to_match_ordered_list", "expect_table_row_count_to_be_between", "expect_compound_columns_to_be_unique", } assert expected_expectations == expectations_from_suite profiler_with_single_column_key = UserConfigurableProfiler( taxi_validator, ignored_columns=ignored_columns, primary_or_compound_key=["pickup_datetime"], ) suite = profiler_with_single_column_key.build_suite() assert len(suite.expectations) == 3 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) expected_expectations = { "expect_table_columns_to_match_ordered_list", "expect_table_row_count_to_be_between", "expect_column_values_to_be_unique", } assert expected_expectations == expectations_from_suite
pg_runtime_batch_request = RuntimeBatchRequest( datasource_name="my_postgresql_datasource", data_connector_name="default_runtime_data_connector_name", data_asset_name="postgres_asset", runtime_parameters={"query": "SELECT * from taxi_data LIMIT 10"}, batch_identifiers={"batch_id": "default_identifier"}, ) # </snippet> # <snippet> validator = context.get_validator(batch_request=mysql_runtime_batch_request, ) # </snippet> # <snippet> profiler = UserConfigurableProfiler( profile_dataset=validator, excluded_expectations=[ "expect_column_quantile_values_to_be_between", "expect_column_mean_to_be_between", ], ) # </snippet> # <snippet> expectation_suite_name = "compare_two_tables" suite = profiler.build_suite() context.save_expectation_suite(expectation_suite=suite, expectation_suite_name=expectation_suite_name) # </snippet> # <snippet> my_checkpoint_name = "comparison_checkpoint" yaml_config = f""" name: {my_checkpoint_name}
def test_profiler_all_expectation_types(titanic_data_context, possible_expectations_set): """ What does this test do and why? Ensures that all available expectation types work as expected """ context = titanic_data_context df = ge.read_csv( file_relative_path( __file__, "../test_sets/taxi_yellow_trip_data_samples/yellow_trip_data_sample_2019-01.csv", )) batch_df = ge.dataset.PandasDataset(df) ignored_columns = [ "pickup_location_id", "dropoff_location_id", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "improvement_surcharge", "congestion_surcharge", ] semantic_types = { "datetime": ["pickup_datetime", "dropoff_datetime"], "numeric": ["total_amount", "passenger_count"], "value_set": [ "payment_type", "rate_code_id", "store_and_fwd_flag", "passenger_count", ], "boolean": ["store_and_fwd_flag"], } profiler = UserConfigurableProfiler( batch_df, semantic_types_dict=semantic_types, ignored_columns=ignored_columns, primary_or_compound_key=[ "vendor_id", "pickup_datetime", "dropoff_datetime", "trip_distance", "pickup_location_id", "dropoff_location_id", ], ) assert profiler.column_info.get("rate_code_id") suite = profiler.build_suite() assert len(suite.expectations) == 46 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) unexpected_expectations = { "expect_column_values_to_be_unique", "expect_column_values_to_be_null", } assert expectations_from_suite == { i for i in possible_expectations_set if i not in unexpected_expectations } ignored_included_columns_overlap = [ i for i in columns_with_expectations if i in ignored_columns ] assert len(ignored_included_columns_overlap) == 0 results = context.run_validation_operator("action_list_operator", assets_to_validate=[batch_df]) assert results["success"]
def test_build_suite_when_suite_already_exists( mock_emit, cardinality_validator, ): """ What does this test do and why? Confirms that creating a new suite on an existing profiler wipes the previous suite """ profiler = UserConfigurableProfiler( cardinality_validator, table_expectations_only=True, excluded_expectations=["expect_table_row_count_to_be_between"], ) suite = profiler.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite(suite) assert len(suite.expectations) == 1 assert "expect_table_columns_to_match_ordered_list" in expectations profiler.excluded_expectations = ["expect_table_columns_to_match_ordered_list"] suite = profiler.build_suite() _, expectations = get_set_of_columns_and_expectations_from_suite(suite) assert len(suite.expectations) == 1 assert "expect_table_row_count_to_be_between" in expectations assert mock_emit.call_count == 2 # noinspection PyUnresolvedReferences expected_events: List[unittest.mock._Call] # noinspection PyUnresolvedReferences actual_events: List[unittest.mock._Call] expected_events = [ mock.call( { "event": "legacy_profiler.build_suite", "event_payload": { "profile_dataset_type": "Validator", "excluded_expectations_specified": True, "ignored_columns_specified": True, "not_null_only": False, "primary_or_compound_key_specified": False, "semantic_types_dict_specified": False, "table_expectations_only": True, "value_set_threshold_specified": True, "api_version": "v2", }, "success": True, } ), mock.call( { "event": "legacy_profiler.build_suite", "event_payload": { "profile_dataset_type": "Validator", "excluded_expectations_specified": True, "ignored_columns_specified": True, "not_null_only": False, "primary_or_compound_key_specified": False, "semantic_types_dict_specified": False, "table_expectations_only": True, "value_set_threshold_specified": True, "api_version": "v2", }, "success": True, } ), ] actual_events = mock_emit.call_args_list assert actual_events == expected_events
"fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "improvement_surcharge", "total_amount", "congestion_surcharge", ] # </snippet> profiler = UserConfigurableProfiler( profile_dataset=validator, excluded_expectations=None, ignored_columns=exclude_column_names, not_null_only=False, primary_or_compound_key=None, semantic_types_dict=None, table_expectations_only=False, value_set_threshold="MANY", ) suite = profiler.build_suite() validator.expectation_suite = suite validator.save_expectation_suite(discard_failed_expectations=False) # Create first checkpoint on yellow_tripdata_sample_2019-01.csv my_checkpoint_config = f""" name: getting_started_checkpoint config_version: 1.0 class_name: SimpleCheckpoint run_name_template: "%Y%m%d-%H%M%S-my-run-name-template" validations:
def test_error_handling_for_expect_compound_columns_to_be_unique( taxi_validator_pandas, taxi_data_ignored_columns, caplog): # TODO: When this expectation is implemented for V3, remove this test and test for this expectation ignored_columns = taxi_data_ignored_columns + [ "pickup_datetime", "dropoff_datetime", "total_amount", "passenger_count", "payment_type", "rate_code_id", "store_and_fwd_flag", "passenger_count", "store_and_fwd_flag", "vendor_id", "trip_distance", ] profiler = UserConfigurableProfiler( taxi_validator_pandas, ignored_columns=ignored_columns, primary_or_compound_key=[ "vendor_id", "pickup_datetime", "dropoff_datetime", "trip_distance", "pickup_location_id", "dropoff_location_id", ], ) with caplog.at_level(logging.WARNING): suite = profiler.build_suite() log_warnings = caplog.messages assert len(log_warnings) == 1 assert ( log_warnings[0] == "expect_compound_columns_to_be_unique is not currently available in the V3 (Batch Request) API. Specifying a compound key will not add any expectations. This will be updated when that expectation becomes available." ) assert len(suite.expectations) == 2 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) expected_expectations = { "expect_table_columns_to_match_ordered_list", "expect_table_row_count_to_be_between", } assert expected_expectations == expectations_from_suite profiler_with_single_column_key = UserConfigurableProfiler( taxi_validator_pandas, ignored_columns=ignored_columns, primary_or_compound_key=["pickup_datetime"], ) suite = profiler_with_single_column_key.build_suite() assert len(suite.expectations) == 3 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) expected_expectations = { "expect_table_columns_to_match_ordered_list", "expect_table_row_count_to_be_between", "expect_column_values_to_be_unique", } assert expected_expectations == expectations_from_suite
def test_profiler_all_expectation_types_sqlalchemy( titanic_data_context_modular_api, taxi_validator_sqlalchemy, possible_expectations_set, taxi_data_semantic_types, taxi_data_ignored_columns, ): """ What does this test do and why? Ensures that all available expectation types work as expected for sqlalchemy """ if taxi_validator_sqlalchemy is None: pytest.skip("a message") context = titanic_data_context_modular_api profiler = UserConfigurableProfiler( taxi_validator_sqlalchemy, semantic_types_dict=taxi_data_semantic_types, ignored_columns=taxi_data_ignored_columns, # TODO: Add primary_or_compound_key test # primary_or_compound_key=[ # "vendor_id", # "pickup_datetime", # "dropoff_datetime", # "trip_distance", # "pickup_location_id", # "dropoff_location_id", # ], ) assert profiler.column_info.get("rate_code_id") with pytest.deprecated_call( ): # parse_strings_as_datetimes is deprecated in V3 suite = profiler.build_suite() assert len(suite.expectations) == 45 ( columns_with_expectations, expectations_from_suite, ) = get_set_of_columns_and_expectations_from_suite(suite) unexpected_expectations = { "expect_column_values_to_be_unique", "expect_column_values_to_be_null", "expect_compound_columns_to_be_unique", } assert expectations_from_suite == { i for i in possible_expectations_set if i not in unexpected_expectations } ignored_included_columns_overlap = [ i for i in columns_with_expectations if i in taxi_data_ignored_columns ] assert len(ignored_included_columns_overlap) == 0 with pytest.deprecated_call( ): # parse_strings_as_datetimes is deprecated in V3 results = context.run_validation_operator( "action_list_operator", assets_to_validate=[taxi_validator_sqlalchemy]) assert results["success"]