def metrics_parameter_builders_by_domain( self, ) -> Dict[Domain, List[ParameterBuilder]]: table_row_count_metric_multi_batch_parameter_builder: MetricMultiBatchParameterBuilder = MetricMultiBatchParameterBuilder( name="table_row_count", metric_name="table.row_count", metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME, metric_value_kwargs=None, enforce_numeric_metric=True, replace_nan_with_zero=True, reduce_scalar_metric=True, evaluation_parameter_builder_configs=None, json_serialize=True, data_context=None, ) column_distinct_values_metric_multi_batch_parameter_builder: MetricMultiBatchParameterBuilder = MetricMultiBatchParameterBuilder( name="column_distinct_values.count", metric_name="column.distinct_values.count", metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME, metric_value_kwargs=None, enforce_numeric_metric=True, replace_nan_with_zero=True, reduce_scalar_metric=True, evaluation_parameter_builder_configs=None, json_serialize=True, data_context=None, ) return { Domain(domain_type=MetricDomainTypes.TABLE,): [ table_row_count_metric_multi_batch_parameter_builder, ], Domain(domain_type=MetricDomainTypes.COLUMN,): [ column_distinct_values_metric_multi_batch_parameter_builder, ], }
def test_simple_date_format_parameter_builder_zero_batch_id_error( alice_columnar_table_single_batch_context, ): data_context: DataContext = alice_columnar_table_single_batch_context date_format_string_parameter: ParameterBuilder = ( SimpleDateFormatStringParameterBuilder( name="my_simple_date_format_string_parameter_builder", data_context=data_context, )) domain: Domain = Domain( domain_type=MetricDomainTypes.COLUMN, rule_name="my_rule", ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } with pytest.raises(ge_exceptions.ProfilerExecutionError) as e: date_format_string_parameter.build_parameters( domain=domain, parameters=parameters, ) assert ( str(e.value) == "Utilizing a SimpleDateFormatStringParameterBuilder requires a non-empty list of Batch identifiers." )
def table_Users_domain(): return Domain( domain_type=MetricDomainTypes.TABLE, domain_kwargs=None, details=None, rule_name="my_rule", )
def test_default_expectation_configuration_builder_alice_null_condition_parameter_builder_validation_dependency_included( alice_columnar_table_single_batch_context, ): data_context: DataContext = alice_columnar_table_single_batch_context batch_request: dict = { "datasource_name": "alice_columnar_table_single_batch_datasource", "data_connector_name": "alice_columnar_table_single_batch_data_connector", "data_asset_name": "alice_columnar_table_single_batch_data_asset", } metric_domain_kwargs: dict = {"column": "user_id"} parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) domain: Domain = Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs=metric_domain_kwargs, rule_name="my_rule", ) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } fully_qualified_parameter_name_for_value: str = "$parameter.my_min_user_id.value[0]" condition: Optional[str] = None max_user_id: int = 999999999999 min_user_id_parameter_builder_config: ParameterBuilderConfig = ( ParameterBuilderConfig( module_name= "great_expectations.rule_based_profiler.parameter_builder", class_name="MetricMultiBatchParameterBuilder", name="my_min_user_id", metric_name="column.min", metric_domain_kwargs=metric_domain_kwargs, )) validation_parameter_builder_configs: Optional[ List[ParameterBuilderConfig]] = [ min_user_id_parameter_builder_config, ] default_expectation_configuration_builder = DefaultExpectationConfigurationBuilder( expectation_type="expect_column_values_to_be_between", condition=condition, min_value=fully_qualified_parameter_name_for_value, max_value=max_user_id, validation_parameter_builder_configs= validation_parameter_builder_configs, data_context=data_context, ) expectation_configuration: Optional[ ExpectationConfiguration] = default_expectation_configuration_builder.build_expectation_configuration( domain=domain, parameters=parameters, batch_request=batch_request, ) assert expectation_configuration.kwargs["min_value"] == 397433
def test_onboarding_data_assistant_metrics_count( bobby_onboarding_data_assistant_result: OnboardingDataAssistantResult, ) -> None: domain: Domain parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] num_metrics: int domain_key: Domain = Domain( domain_type=MetricDomainTypes.TABLE, ) num_metrics = 0 for ( domain, parameter_values_for_fully_qualified_parameter_names, ) in bobby_onboarding_data_assistant_result.metrics_by_domain.items(): if domain.is_superset(domain_key): num_metrics += len(parameter_values_for_fully_qualified_parameter_names) assert num_metrics == 2 num_metrics = 0 for ( domain, parameter_values_for_fully_qualified_parameter_names, ) in bobby_onboarding_data_assistant_result.metrics_by_domain.items(): num_metrics += len(parameter_values_for_fully_qualified_parameter_names) assert num_metrics == 184
def _get_domains( self, variables: Optional[ParameterContainer] = None, ) -> List[Domain]: domains: List[Domain] = [Domain(domain_type=self.domain_type, )] return domains
def table_Users_domain(): skip_if_python_below_minimum_version() return Domain( domain_type=MetricDomainTypes.TABLE, domain_kwargs=None, details=None, )
def test_bootstrap_numeric_metric_range_multi_batch_parameter_builder_bobby_false_positive_rate_very_small( bobby_columnar_table_multi_batch_deterministic_data_context, ): data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) # BatchRequest yielding three batches batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } # a commonly used defect rate in quality control that equates to 3.4 defects per million opportunities six_sigma_false_positive_rate: float = 3.4 / 1000000.0 assert six_sigma_false_positive_rate > NP_EPSILON # what if user tries a false positive rate smaller than NP_EPSILON (by an order of magnitude in this case)? smaller_than_np_epsilon_false_positive_rate: float = NP_EPSILON / 10 numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="row_count_range", metric_name="table.row_count", metric_multi_batch_parameter_builder_name=None, metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME, metric_value_kwargs=None, estimator="bootstrap", false_positive_rate=smaller_than_np_epsilon_false_positive_rate, round_decimals=0, evaluation_parameter_builder_configs=None, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( domain_type=MetricDomainTypes.TABLE, rule_name="my_rule", ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None warning_message: str = re.escape( f"""You have chosen a false_positive_rate of {smaller_than_np_epsilon_false_positive_rate}, which is too close to 0. A false_positive_rate of {NP_EPSILON} has been selected instead.""") with pytest.warns(UserWarning, match=warning_message): numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, )
def test_simple_date_format_parameter_builder_bobby( bobby_columnar_table_multi_batch_deterministic_data_context, ): data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) metric_domain_kwargs: dict = {"column": "pickup_datetime"} candidate_strings: set[str] = { "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", } threshold: float = 0.9 batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } date_format_string_parameter: SimpleDateFormatStringParameterBuilder = ( SimpleDateFormatStringParameterBuilder( name="my_simple_date_format_string_parameter_builder", metric_domain_kwargs=metric_domain_kwargs, candidate_strings=candidate_strings, threshold=threshold, data_context=data_context, batch_request=batch_request, )) assert date_format_string_parameter.CANDIDATE_STRINGS != candidate_strings assert date_format_string_parameter._candidate_strings == candidate_strings assert date_format_string_parameter._threshold == 0.9 parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) domain: Domain = Domain(domain_type=MetricDomainTypes.COLUMN, domain_kwargs=metric_domain_kwargs) assert parameter_container.parameter_nodes is None date_format_string_parameter._build_parameters( parameter_container=parameter_container, domain=domain) assert len(parameter_container.parameter_nodes) == 1 fully_qualified_parameter_name_for_value: str = ( "$parameter.my_simple_date_format_string_parameter_builder") expected_value: dict = { "value": "%Y-%m-%d %H:%M:%S", "details": { "success_ratio": 1.0 }, } assert (get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name=fully_qualified_parameter_name_for_value, domain=domain, parameters={domain.id: parameter_container}, ) == expected_value)
def column_Date_domain(): return Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs={ "column": "Date", }, details=None, rule_name="my_rule", )
def column_pair_Age_Date_domain(): return Domain( domain_type=MetricDomainTypes.COLUMN_PAIR, domain_kwargs={ "column_A": "Age", "column_B": "Date", }, details=None, rule_name="my_rule", )
def column_Date_domain(): skip_if_python_below_minimum_version() return Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs={ "column": "Date", "batch_id": "c260e179bb1bc81d84bba72a8110d8e2", }, details=None, )
def test_get_fully_qualified_parameter_names( parameters_with_different_depth_level_values, ): parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None) build_parameter_container( parameter_container=parameter_container, parameter_values=parameters_with_different_depth_level_values, ) domain: Domain = Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs=None, details=None, rule_name="my_rule", ) # Convert variables argument to ParameterContainer variables: ParameterContainer = build_parameter_container_for_variables( variables_configs={ "my_int": 9, "my_float": 3.38, "my_string": "hello", } ) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } expected_fully_qualified_parameter_names: List[str] = [ "$variables", "$parameter.date_strings.yyyy_mm_dd_hh_mm_ss_tz_date_format", "$parameter.date_strings.yyyy_mm_dd_date_format", "$parameter.date_strings.mm_yyyy_dd_hh_mm_ss_tz_date_format", "$parameter.date_strings.mm_yyyy_dd_date_format", "$parameter.date_strings.tolerances.max_abs_error_time_milliseconds", "$parameter.date_strings.tolerances.max_num_conversion_attempts", "$parameter.tolerances.mostly", "$parameter.tolerances.financial.usd", "$parameter.monthly_taxi_fairs.mean_values", "$parameter.daily_taxi_fairs.mean_values", "$parameter.weekly_taxi_fairs.mean_values", "$mean", ] fully_qualified_parameter_names: List[str] = get_fully_qualified_parameter_names( domain=domain, variables=variables, parameters=parameters, ) assert len(fully_qualified_parameter_names) == len( expected_fully_qualified_parameter_names ) assert sorted(fully_qualified_parameter_names) == sorted( expected_fully_qualified_parameter_names )
def test_bootstrap_numeric_metric_range_multi_batch_parameter_builder_bobby_false_positive_rate_negative( bobby_columnar_table_multi_batch_deterministic_data_context, ): data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) # BatchRequest yielding three batches batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="row_count_range", metric_name="table.row_count", metric_multi_batch_parameter_builder_name=None, metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME, metric_value_kwargs=None, estimator="bootstrap", false_positive_rate=-0.05, round_decimals=0, evaluation_parameter_builder_configs=None, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( domain_type=MetricDomainTypes.TABLE, rule_name="my_rule", ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None error_message: str = re.escape( """false_positive_rate must be a positive decimal number between 0 and 1 inclusive [0, 1], but -0.05 was provided.""") with pytest.raises(ge_exceptions.ProfilerExecutionError, match=error_message): numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, )
def multi_column_Age_Date_Description_domain(): return Domain( domain_type=MetricDomainTypes.MULTICOLUMN, domain_kwargs={ "column_list": [ "Age", "Date", "Description", ], }, details=None, rule_name="my_rule", )
def test_bootstrap_numeric_metric_range_multi_batch_parameter_builder_bobby_false_positive_rate_zero( bobby_columnar_table_multi_batch_deterministic_data_context, ): data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) # BatchRequest yielding three batches batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="row_count_range", metric_name="table.row_count", metric_multi_batch_parameter_builder_name=None, metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME, metric_value_kwargs=None, estimator="bootstrap", false_positive_rate=0.0, round_decimals=0, evaluation_parameter_builder_configs=None, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( domain_type=MetricDomainTypes.TABLE, rule_name="my_rule", ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None warning_message: str = re.escape( f"""You have chosen a false_positive_rate of 0.0, which is too close to 0. A false_positive_rate of {NP_EPSILON} has been selected instead.""") with pytest.warns(UserWarning, match=warning_message): numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, )
def test_simple_date_format_parameter_builder_alice( alice_columnar_table_single_batch_context, ): data_context: DataContext = alice_columnar_table_single_batch_context batch_request: dict = { "datasource_name": "alice_columnar_table_single_batch_datasource", "data_connector_name": "alice_columnar_table_single_batch_data_connector", "data_asset_name": "alice_columnar_table_single_batch_data_asset", } metric_domain_kwargs = {"column": "event_ts"} date_format_string_parameter: SimpleDateFormatStringParameterBuilder = ( SimpleDateFormatStringParameterBuilder( name="my_date_format", metric_domain_kwargs=metric_domain_kwargs, data_context=data_context, batch_request=batch_request, )) assert date_format_string_parameter.CANDIDATE_STRINGS == DEFAULT_CANDIDATE_STRINGS assert date_format_string_parameter.candidate_strings is None assert date_format_string_parameter._threshold == 1.0 parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) domain: Domain = Domain(domain_type=MetricDomainTypes.COLUMN, domain_kwargs=metric_domain_kwargs) assert parameter_container.parameter_nodes is None date_format_string_parameter._build_parameters( parameter_container=parameter_container, domain=domain) # noinspection PyTypeChecker assert len(parameter_container.parameter_nodes) == 1 fully_qualified_parameter_name_for_value: str = "$parameter.my_date_format" expected_value: dict = { "value": "%Y-%m-%d %H:%M:%S", "details": { "success_ratio": 1.0 }, } assert (get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name=fully_qualified_parameter_name_for_value, domain=domain, parameters={domain.id: parameter_container}, ) == expected_value)
def test_column_values_unique_single_batch( alice_columnar_table_single_batch_context): data_context: DataContext = alice_columnar_table_single_batch_context batch_request: BatchRequest = BatchRequest( datasource_name="alice_columnar_table_single_batch_datasource", data_connector_name="alice_columnar_table_single_batch_data_connector", data_asset_name="alice_columnar_table_single_batch_data_asset", ) domain_builder: MapMetricColumnDomainBuilder = MapMetricColumnDomainBuilder( map_metric_name="column_values.unique", max_unexpected_values=0, max_unexpected_ratio=None, min_max_unexpected_values_proportion=9.75e-1, data_context=data_context, ) domains: List[Domain] = domain_builder.get_domains( rule_name="my_rule", batch_request=batch_request) # Unit Tests for "inferred_semantic_domain_type" are provided separately. domain: Domain for domain in domains: domain.details = {} domains = sorted(domains, key=lambda x: x.domain_kwargs["column"]) alice_compliant_column_names: List[str] = [ "id", "event_type", "user_id", "event_ts", "server_ts", "device_ts", ] column_name: str alice_expected_column_domains: List[Domain] = [ Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs={ "column": column_name, }, rule_name="my_rule", ) for column_name in alice_compliant_column_names ] alice_expected_column_domains = sorted( alice_expected_column_domains, key=lambda x: x.domain_kwargs["column"]) assert len(domains) == 6 assert domains == alice_expected_column_domains
def _get_domains( self, rule_name: str, variables: Optional[ParameterContainer] = None, ) -> List[Domain]: """Return domains matching the specified tolerance limits. Args: rule_name: name of Rule object, for which "Domain" objects are obtained. variables: Optional variables to substitute when evaluating. Returns: List of domains that match the desired tolerance limits. """ batch_ids: List[str] = self.get_batch_ids(variables=variables) validator: "Validator" = self.get_validator(variables=variables) # noqa: F821 effective_column_names: List[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, ) if not (self.include_column_names and effective_column_names): raise ge_exceptions.ProfilerExecutionError( message=f'Error: "column_list" in {self.__class__.__name__} must not be empty.' ) column_name: str semantic_types_by_column_name: Dict[str, SemanticDomainTypes] = { column_name: self.semantic_type_filter.table_column_name_to_inferred_semantic_domain_type_map[ column_name ] for column_name in effective_column_names } domains: List[Domain] = [ Domain( domain_type=self.domain_type, domain_kwargs={ "column_list": effective_column_names, }, details={ INFERRED_SEMANTIC_TYPE_KEY: semantic_types_by_column_name, }, rule_name=rule_name, ), ] return domains
def test_simple_date_format_parameter_builder_zero_batch_id_error(): date_format_string_parameter: SimpleDateFormatStringParameterBuilder = ( SimpleDateFormatStringParameterBuilder( name="my_simple_date_format_string_parameter_builder", )) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) domain: Domain = Domain(domain_type=MetricDomainTypes.COLUMN) with pytest.raises(ge_exceptions.ProfilerExecutionError) as e: date_format_string_parameter._build_parameters( parameter_container=parameter_container, domain=domain) assert ( str(e.value) == "Utilizing a SimpleDateFormatStringParameterBuilder requires a non-empty list of batch identifiers." )
def test_excluded_columns_single_batch( alice_columnar_table_single_batch_context): data_context: DataContext = alice_columnar_table_single_batch_context batch_request: BatchRequest = BatchRequest( datasource_name="alice_columnar_table_single_batch_datasource", data_connector_name="alice_columnar_table_single_batch_data_connector", data_asset_name="alice_columnar_table_single_batch_data_asset", ) domain_builder: DomainBuilder = CategoricalColumnDomainBuilder( limit_mode="VERY_FEW", exclude_column_names=[ "id", "event_type", "user_id", "event_ts", "server_ts", ], data_context=data_context, ) domains: List[Domain] = domain_builder.get_domains( rule_name="my_rule", batch_request=batch_request) alice_all_column_names: List[str] = [ "device_ts", "user_agent", ] column_name: str alice_all_column_domains: List[Domain] = [ Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs={ "column": column_name, }, rule_name="my_rule", ) for column_name in alice_all_column_names ] assert len(domains) == 2 # Unit Tests for "inferred_semantic_domain_type" are provided separately. domain: Domain for domain in domains: domain.details = {} assert domains == alice_all_column_domains
def __init__( self, name: str, validator: Optional[Validator], ) -> None: """ DataAssistant subclasses guide "RuleBasedProfiler" to contain Rule configurations to embody profiling behaviors, corresponding to indended exploration and validation goals. Then executing "RuleBasedProfiler.run()" yields "RuleBasedProfilerResult" object, containing "fully_qualified_parameter_names_by_domain", "parameter_values_for_fully_qualified_parameter_names_by_domain", "expectation_configurations", and "citation", immediately available for composing "ExpectationSuite" and validating underlying data "Batch" objects. Args: name: the name of this DataAssistant object validator: Validator object, containing loaded Batch objects as well as Expectation and Metric operations """ self._name = name self._validator = validator if validator is None: self._data_context = None self._batches = None else: self._data_context = self._validator.data_context self._batches = self._validator.batches variables: Optional[Dict[str, Any]] = self.get_variables() or {} self._profiler = RuleBasedProfiler( name=self.name, config_version=1.0, variables=variables, data_context=self._data_context, ) self._metrics_parameter_builders_by_domain = {} rules: Optional[List[Rule]] = self.get_rules() or [] rule: Rule for rule in rules: self.profiler.add_rule(rule=rule) self._metrics_parameter_builders_by_domain[Domain( domain_type=rule.domain_builder.domain_type, rule_name=rule.name, )] = rule.parameter_builders
def _get_domains( self, variables: Optional[ParameterContainer] = None, ) -> List[Domain]: """ Find the column suffix for each column and return all domains matching the specified suffix. """ column_name_suffixes: Union[str, Iterable, List[str]] = self.column_name_suffixes if isinstance(column_name_suffixes, str): column_name_suffixes = [column_name_suffixes] else: if not isinstance(column_name_suffixes, (Iterable, List)): raise ValueError( "Unrecognized column_name_suffixes directive -- must be a list or a string." ) batch_id: str = self.get_batch_id(variables=variables) table_column_names: List[str] = self.get_validator( variables=variables).get_metric(metric=MetricConfiguration( metric_name="table.columns", metric_domain_kwargs={ "batch_id": batch_id, }, metric_value_kwargs=None, metric_dependencies=None, )) candidate_column_names: List[str] = list( filter( lambda candidate_column_name: candidate_column_name.endswith( tuple(column_name_suffixes)), table_column_names, )) column_name: str domains: List[Domain] = [ Domain( domain_type=self.domain_type, domain_kwargs={ "column": column_name, }, ) for column_name in candidate_column_names ] return domains
def test_regex_wrong_domain(mock_data_context: mock.MagicMock, batch_fixture: Batch): batch: Batch = batch_fixture mock_data_context.get_batch_list.return_value = [batch] mock_data_context.get_validator_using_batch_list.return_value = Validator( execution_engine=PandasExecutionEngine(), batches=[batch] ) data_context: DataContext = mock_data_context # column : c does not exist metric_domain_kwargs: dict = {"column": "c"} candidate_regexes: List[str] = [r"^\d{1}$"] regex_pattern_string_parameter_builder: ParameterBuilder = ( RegexPatternStringParameterBuilder( name="my_regex_pattern_string_parameter_builder", metric_domain_kwargs=metric_domain_kwargs, candidate_regexes=candidate_regexes, data_context=data_context, ) ) domain: Domain = Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs=metric_domain_kwargs, rule_name="my_rule", ) parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } with pytest.raises(ge_exceptions.ProfilerExecutionError) as e: regex_pattern_string_parameter_builder.build_parameters( domain=domain, parameters=parameters, batch_list=[batch], ) assert ( e.value.message == "Result of metric computations for RegexPatternStringParameterBuilder is empty." )
def test_semantic_domain_consistency(): domain: Domain with pytest.raises(ValueError) as excinfo: # noinspection PyUnusedLocal domain = Domain( domain_type="column", domain_kwargs={"column": "passenger_count"}, details={ "estimator": "categorical", "cardinality": "low", INFERRED_SEMANTIC_TYPE_KEY: { "num_passengers": SemanticDomainTypes.NUMERIC, }, }, rule_name="my_rule", ) assert ( """Cannot instantiate Domain (domain_type "MetricDomainTypes.COLUMN" of type "<enum 'MetricDomainTypes'>" -- key "num_passengers", detected in "inferred_semantic_domain_type" dictionary, does not exist as value of appropriate key in "domain_kwargs" dictionary.""" in str(excinfo.value))
def _get_domains( self, variables: Optional[ParameterContainer] = None, ) -> List[Domain]: """ Obtains and returns domains for all columns of a table (or for configured columns, if they exist in the table). """ batch_id: str = self.get_batch_id(variables=variables) table_columns: List[str] = self.get_validator(variables=variables).get_metric( metric=MetricConfiguration( metric_name="table.columns", metric_domain_kwargs={ "batch_id": batch_id, }, metric_value_kwargs=None, metric_dependencies=None, ) ) if self.column_names is None: self.column_names = table_columns else: column_name: str for column_name in self.column_names: if column_name not in table_columns: raise ge_exceptions.ProfilerExecutionError( message=f'Error: The column "{column_name}" in BatchData does not exist.' ) column_name: str domains: List[Domain] = [ Domain( domain_type=self.domain_type, domain_kwargs={ "column": column_name, }, ) for column_name in self.column_names ] return domains
def _get_domains( self, variables: Optional[ParameterContainer] = None, ) -> List[Domain]: """ Find the semantic column type for each column and return all domains matching the specified type or types. """ batch_id: str = self.get_batch_id(variables=variables) table_column_names: List[str] = self.get_validator( variables=variables).get_metric(metric=MetricConfiguration( metric_name="table.columns", metric_domain_kwargs={ "batch_id": batch_id, }, metric_value_kwargs=None, metric_dependencies=None, )) # First check the column name ends in "_id". candidate_column_names: List[str] = list( filter( lambda candidate_column_name: candidate_column_name.endswith( tuple(self.column_name_suffixes)), table_column_names, )) column_name: str domains: List[Domain] = [ Domain( domain_type=MetricDomainTypes.COLUMN, domain_kwargs={ "column": column_name, }, ) for column_name in candidate_column_names ] return domains
def build_domains_from_column_names( rule_name: str, column_names: List[str], domain_type: MetricDomainTypes, table_column_name_to_inferred_semantic_domain_type_map: Optional[Dict[ str, SemanticDomainTypes]] = None, ) -> List[Domain]: """ This utility method builds "simple" Domain objects (i.e., required fields only, no "details" metadata accepted). :param rule_name: name of Rule object, for which "Domain" objects are obtained. :param column_names: list of column names to serve as values for "column" keys in "domain_kwargs" dictionary :param domain_type: type of Domain objects (same "domain_type" must be applicable to all Domain objects returned) :param table_column_name_to_inferred_semantic_domain_type_map: map from column name to inferred semantic type :return: list of resulting Domain objects """ column_name: str domains: List[Domain] = [ Domain( domain_type=domain_type, domain_kwargs={ "column": column_name, }, details={ INFERRED_SEMANTIC_TYPE_KEY: { column_name: table_column_name_to_inferred_semantic_domain_type_map[ column_name], } if table_column_name_to_inferred_semantic_domain_type_map else None, }, rule_name=rule_name, ) for column_name in column_names ] return domains
def test_numeric_metric_range_multi_batch_parameter_builder_bobby_kde_bw_method( bobby_columnar_table_multi_batch_deterministic_data_context, ): """ This tests whether a change to bw_method results in a change to the range """ data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) # BatchRequest yielding three batches batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } metric_domain_kwargs: dict = {"column": "fare_amount"} numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="column_min_range", metric_name="column.min", metric_domain_kwargs=metric_domain_kwargs, estimator="kde", false_positive_rate=5.0e-2, round_decimals=0, json_serialize=False, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( rule_name="my_rule", domain_type=MetricDomainTypes.TABLE, ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, ) parameter_nodes: Optional[Dict[str, ParameterNode]] = ( parameter_container.parameter_nodes or {}) assert len(parameter_nodes) == 1 fully_qualified_parameter_name_for_value: str = "$parameter.column_min_range" parameter_node: ParameterNode = ( get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name= fully_qualified_parameter_name_for_value, domain=domain, parameters=parameters, )) default_bw_method_value: np.ndarray = parameter_node.pop("value") numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="column_min_range", metric_name="column.min", metric_domain_kwargs=metric_domain_kwargs, estimator="kde", bw_method=0.5, false_positive_rate=5.0e-2, round_decimals=0, json_serialize=False, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( rule_name="my_rule", domain_type=MetricDomainTypes.TABLE, ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, ) parameter_nodes: Optional[Dict[str, ParameterNode]] = ( parameter_container.parameter_nodes or {}) assert len(parameter_nodes) == 1 fully_qualified_parameter_name_for_value: str = "$parameter.column_min_range" parameter_node: ParameterNode = ( get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name= fully_qualified_parameter_name_for_value, domain=domain, parameters=parameters, )) other_bw_method_value: np.ndarray = parameter_node.pop("value") assert default_bw_method_value[0] != other_bw_method_value[0]
def test_kde_numeric_metric_range_multi_batch_parameter_builder_bobby( bobby_columnar_table_multi_batch_deterministic_data_context, ): data_context: DataContext = ( bobby_columnar_table_multi_batch_deterministic_data_context) # BatchRequest yielding three batches batch_request: dict = { "datasource_name": "taxi_pandas", "data_connector_name": "monthly", "data_asset_name": "my_reports", } numeric_metric_range_parameter_builder: ParameterBuilder = ( NumericMetricRangeMultiBatchParameterBuilder( name="row_count_range", metric_name="table.row_count", estimator="kde", include_estimator_samples_histogram_in_details=True, false_positive_rate=1.0e-2, round_decimals=0, json_serialize=False, data_context=data_context, )) variables: Optional[ParameterContainer] = None domain: Domain = Domain( rule_name="my_rule", domain_type=MetricDomainTypes.TABLE, ) parameter_container: ParameterContainer = ParameterContainer( parameter_nodes=None) parameters: Dict[str, ParameterContainer] = { domain.id: parameter_container, } assert parameter_container.parameter_nodes is None numeric_metric_range_parameter_builder.build_parameters( domain=domain, variables=variables, parameters=parameters, batch_request=batch_request, ) parameter_nodes: Optional[Dict[str, ParameterNode]] = ( parameter_container.parameter_nodes or {}) assert len(parameter_nodes) == 1 fully_qualified_parameter_name_for_value: str = "$parameter.row_count_range" expected_value_dict: dict = { "value": None, "details": { "metric_configuration": { "domain_kwargs": {}, "metric_name": "table.row_count", "metric_value_kwargs": None, "metric_dependencies": None, }, "num_batches": 3, }, } parameter_node: ParameterNode = ( get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name= fully_qualified_parameter_name_for_value, domain=domain, parameters=parameters, )) actual_value: np.ndarray = parameter_node.pop("value") parameter_node["value"] = None actual_estimation_histogram: np.ndarray = parameter_node.details.pop( "estimation_histogram") assert parameter_node == expected_value_dict expected_value: np.ndarray = np.array([6180, 10277]) # Measure of "closeness" between "actual" and "desired" is computed as: atol + rtol * abs(desired) # (see "https://numpy.org/doc/stable/reference/generated/numpy.testing.assert_allclose.html" for details). rtol: float = 1.0e-2 atol: float = 0 # kde results should be stable +/- 1% np.testing.assert_allclose( actual=actual_value, desired=expected_value, rtol=rtol, atol=atol, err_msg= f"Actual value of {actual_value} differs from expected value of {expected_value} by more than {atol + rtol * abs(expected_value)} tolerance.", ) expected_estimation_histogram: np.ndarray = np.array([ 13.0, 155.0, 719.0, 1546.0, 2221.0, 2570.0, 1946.0, 683.0, 137.0, 9.0, ]) # Assert no significant difference between expected (null hypothesis) and actual estimation histograms. ks_result: tuple = stats.ks_2samp(data1=actual_estimation_histogram, data2=expected_estimation_histogram) p_value: float = ks_result[1] assert p_value > 9.5e-1