def test_build_parameter_container(
parameters_with_different_depth_level_values,
multi_part_name_parameter_container,
):
parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)
build_parameter_container(
parameter_container=parameter_container,
parameter_values=parameters_with_different_depth_level_values,
)
assert parameter_container == multi_part_name_parameter_container
def test_get_fully_qualified_parameter_names(
parameters_with_different_depth_level_values,
):
parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)
build_parameter_container(
parameter_container=parameter_container,
parameter_values=parameters_with_different_depth_level_values,
)
domain: Domain = Domain(
domain_type=MetricDomainTypes.COLUMN,
domain_kwargs=None,
details=None,
rule_name="my_rule",
)
# Convert variables argument to ParameterContainer
variables: ParameterContainer = build_parameter_container_for_variables(
variables_configs={
"my_int": 9,
"my_float": 3.38,
"my_string": "hello",
}
)
parameters: Dict[str, ParameterContainer] = {
domain.id: parameter_container,
}
expected_fully_qualified_parameter_names: List[str] = [
"$variables",
"$parameter.date_strings.yyyy_mm_dd_hh_mm_ss_tz_date_format",
"$parameter.date_strings.yyyy_mm_dd_date_format",
"$parameter.date_strings.mm_yyyy_dd_hh_mm_ss_tz_date_format",
"$parameter.date_strings.mm_yyyy_dd_date_format",
"$parameter.date_strings.tolerances.max_abs_error_time_milliseconds",
"$parameter.date_strings.tolerances.max_num_conversion_attempts",
"$parameter.tolerances.mostly",
"$parameter.tolerances.financial.usd",
"$parameter.monthly_taxi_fairs.mean_values",
"$parameter.daily_taxi_fairs.mean_values",
"$parameter.weekly_taxi_fairs.mean_values",
"$mean",
]
fully_qualified_parameter_names: List[str] = get_fully_qualified_parameter_names(
domain=domain,
variables=variables,
parameters=parameters,
)
assert len(fully_qualified_parameter_names) == len(
expected_fully_qualified_parameter_names
)
assert sorted(fully_qualified_parameter_names) == sorted(
expected_fully_qualified_parameter_names
)
def _build_parameters(
self,
parameter_container: ParameterContainer,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
):
"""
Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and optional
details.
:return: ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and
ptional details
"""
metric_computation_result: MetricComputationResult = self.get_metrics(
metric_name=self.metric_name,
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=self.metric_value_kwargs,
enforce_numeric_metric=self.enforce_numeric_metric,
replace_nan_with_zero=self.replace_nan_with_zero,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values: np.ndarray = metric_computation_result.metric_values
details: MetricComputationDetails = metric_computation_result.details
# Obtain reduce_scalar_metric from "rule state" (i.e., variables and parameters); from instance variable otherwise.
reduce_scalar_metric: bool = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.reduce_scalar_metric,
expected_return_type=bool,
variables=variables,
parameters=parameters,
)
# As a simplification, apply reduction to scalar in case of one-dimensional metric (for convenience).
if reduce_scalar_metric and metric_values.shape[1] == 1:
metric_values = metric_values[:, 0]
parameter_values: Dict[str, Any] = {
f"$parameter.{self.name}": {
"value": metric_values,
"details": details,
},
}
build_parameter_container(parameter_container=parameter_container,
parameter_values=parameter_values)
def test_get_parameter_values_for_fully_qualified_parameter_names(
parameters_with_different_depth_level_values,
):
parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)
build_parameter_container(
parameter_container=parameter_container,
parameter_values=parameters_with_different_depth_level_values,
)
domain: Domain = Domain(
domain_type=MetricDomainTypes.COLUMN,
domain_kwargs=None,
details=None,
rule_name="my_rule",
)
# Convert variables argument to ParameterContainer
variables: ParameterContainer = build_parameter_container_for_variables(
variables_configs={
"my_int": 9,
"my_float": 3.38,
"my_string": "hello",
}
)
parameters: Dict[str, ParameterContainer] = {
domain.id: parameter_container,
}
# fmt: off
expected_parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] = {
"$variables": {
"my_int": 9,
"my_float": 3.38,
"my_string": "hello",
},
"$parameter.weekly_taxi_fairs.mean_values": {
"value": [
{
"sunday": 71.43,
"monday": 74.35,
"tuesday": 42.3,
"wednesday": 42.3,
"thursday": 82.2,
"friday": 78.78,
"saturday": 91.39,
},
{
"sunday": 81.43,
"monday": 84.35,
"tuesday": 52.3,
"wednesday": 43.3,
"thursday": 22.2,
"friday": 98.78,
"saturday": 81.39,
},
{
"sunday": 61.43,
"monday": 34.35,
"tuesday": 82.3,
"wednesday": 72.3,
"thursday": 22.2,
"friday": 38.78,
"saturday": 51.39,
},
{
"sunday": 51.43,
"monday": 64.35,
"tuesday": 72.3,
"wednesday": 82.3,
"thursday": 22.2,
"friday": 98.78,
"saturday": 31.39,
},
{
"sunday": 72.43,
"monday": 77.35,
"tuesday": 46.3,
"wednesday": 47.3,
"thursday": 88.2,
"friday": 79.78,
"saturday": 93.39,
},
{
"sunday": 72.43,
"monday": 73.35,
"tuesday": 41.3,
"wednesday": 49.3,
"thursday": 80.2,
"friday": 78.78,
"saturday": 93.39,
},
{
"sunday": 74.43,
"monday": 78.35,
"tuesday": 49.3,
"wednesday": 43.3,
"thursday": 88.2,
"friday": 72.78,
"saturday": 97.39,
},
{
"sunday": 73.43,
"monday": 72.35,
"tuesday": 40.3,
"wednesday": 40.3,
"thursday": 89.2,
"friday": 77.78,
"saturday": 90.39,
},
{
"sunday": 72.43,
"monday": 73.35,
"tuesday": 45.3,
"wednesday": 44.3,
"thursday": 89.2,
"friday": 77.78,
"saturday": 96.39,
},
{
"sunday": 75.43,
"monday": 74.25,
"tuesday": 42.33,
"wednesday": 42.23,
"thursday": 82.21,
"friday": 78.76,
"saturday": 91.37,
},
{
"sunday": 71.43,
"monday": 74.37,
"tuesday": 42.3,
"wednesday": 42.32,
"thursday": 82.23,
"friday": 78.77,
"saturday": 91.49,
},
{
"sunday": 71.63,
"monday": 74.37,
"tuesday": 42.2,
"wednesday": 42.1,
"thursday": 82.29,
"friday": 78.79,
"saturday": 91.39,
},
{
"sunday": 71.42,
"monday": 74.33,
"tuesday": 42.33,
"wednesday": 42.34,
"thursday": 82.25,
"friday": 78.77,
"saturday": 91.69,
},
{
"sunday": 71.44,
"monday": 72.35,
"tuesday": 42.33,
"wednesday": 42.31,
"thursday": 82.29,
"friday": 78.68,
"saturday": 91.49,
},
{
"sunday": 71.44,
"monday": 74.32,
"tuesday": 42.32,
"wednesday": 42.32,
"thursday": 82.29,
"friday": 78.77,
"saturday": 91.49,
},
{
"sunday": 71.44,
"monday": 74.33,
"tuesday": 42.21,
"wednesday": 42.31,
"thursday": 82.27,
"friday": 78.74,
"saturday": 91.49,
},
{
"sunday": 71.33,
"monday": 74.25,
"tuesday": 42.31,
"wednesday": 42.03,
"thursday": 82.02,
"friday": 78.08,
"saturday": 91.38,
},
{
"sunday": 71.41,
"monday": 74.31,
"tuesday": 42.39,
"wednesday": 42.93,
"thursday": 82.92,
"friday": 78.75,
"saturday": 91.49,
},
{
"sunday": 72.43,
"monday": 73.35,
"tuesday": 42.3,
"wednesday": 32.3,
"thursday": 52.2,
"friday": 88.78,
"saturday": 81.39,
},
{
"sunday": 71.43,
"monday": 74.35,
"tuesday": 32.3,
"wednesday": 92.3,
"thursday": 72.2,
"friday": 74.78,
"saturday": 51.39,
},
{
"sunday": 72.43,
"monday": 64.35,
"tuesday": 52.3,
"wednesday": 42.39,
"thursday": 82.28,
"friday": 78.77,
"saturday": 91.36,
},
{
"sunday": 81.43,
"monday": 94.35,
"tuesday": 62.3,
"wednesday": 52.3,
"thursday": 92.2,
"friday": 88.78,
"saturday": 51.39,
},
{
"sunday": 21.43,
"monday": 34.35,
"tuesday": 42.34,
"wednesday": 62.3,
"thursday": 52.2,
"friday": 98.78,
"saturday": 81.39,
},
{
"sunday": 71.33,
"monday": 74.25,
"tuesday": 42.13,
"wednesday": 42.93,
"thursday": 82.82,
"friday": 78.78,
"saturday": 91.39,
},
{
"sunday": 72.43,
"monday": 73.35,
"tuesday": 44.3,
"wednesday": 45.3,
"thursday": 86.2,
"friday": 77.78,
"saturday": 98.39,
},
{
"sunday": 79.43,
"monday": 78.35,
"tuesday": 47.3,
"wednesday": 46.3,
"thursday": 85.2,
"friday": 74.78,
"saturday": 93.39,
},
{
"sunday": 71.42,
"monday": 74.31,
"tuesday": 42.0,
"wednesday": 42.1,
"thursday": 82.23,
"friday": 65.78,
"saturday": 91.26,
},
{
"sunday": 91.43,
"monday": 84.35,
"tuesday": 42.37,
"wednesday": 42.36,
"thursday": 82.25,
"friday": 78.74,
"saturday": 91.32,
},
{
"sunday": 71.33,
"monday": 74.45,
"tuesday": 42.35,
"wednesday": 42.36,
"thursday": 82.27,
"friday": 26.78,
"saturday": 71.39,
},
{
"sunday": 71.53,
"monday": 73.35,
"tuesday": 43.32,
"wednesday": 42.23,
"thursday": 82.32,
"friday": 78.18,
"saturday": 91.49,
},
{
"sunday": 71.53,
"monday": 74.25,
"tuesday": 52.3,
"wednesday": 52.3,
"thursday": 81.23,
"friday": 78.78,
"saturday": 78.39,
},
],
"details": {
"confidence": "high",
},
},
"$parameter.tolerances.mostly": 0.91,
"$parameter.tolerances.financial.usd": 1.0,
"$parameter.monthly_taxi_fairs.mean_values": {
"value": [
2.3,
9.8,
42.3,
8.1,
38.5,
53.7,
71.43,
16.34,
49.43,
74.35,
51.98,
46.42,
20.01,
69.44,
65.32,
8.83,
55.79,
82.2,
36.93,
83.78,
31.13,
76.93,
67.67,
25.12,
58.04,
79.78,
90.91,
15.26,
61.65,
78.78,
12.99,
],
"details": {
"confidence": "low",
},
},
"$parameter.date_strings.yyyy_mm_dd_hh_mm_ss_tz_date_format": {
"value": "%Y-%m-%d %H:%M:%S %Z",
"details": {
"confidence": 0.78,
},
},
"$parameter.date_strings.yyyy_mm_dd_date_format": {
"value": "%Y-%m-%d",
"details": {
"confidence": 0.78,
},
},
"$parameter.date_strings.tolerances.max_num_conversion_attempts": 5,
"$parameter.date_strings.tolerances.max_abs_error_time_milliseconds": 100,
"$parameter.date_strings.mm_yyyy_dd_hh_mm_ss_tz_date_format": {
"value": "%m-%Y-%d %H:%M:%S %Z",
"details": {
"confidence": 0.78,
},
},
"$parameter.date_strings.mm_yyyy_dd_date_format": {
"value": "%m-%Y-%d",
"details": {
"confidence": 0.78,
},
},
"$parameter.daily_taxi_fairs.mean_values": {
"value": {
"sunday": 71.43,
"monday": 74.35,
"tuesday": 42.3,
"wednesday": 42.3,
"thursday": 82.2,
"friday": 78.78,
"saturday": 91.39,
},
"details": {
"confidence": "medium",
},
},
"$mean": 0.65,
}
# fmt: on
parameter_values_for_fully_qualified_parameter_names: Dict[
str, ParameterNode
] = get_parameter_values_for_fully_qualified_parameter_names(
domain=domain,
variables=variables,
parameters=parameters,
)
assert (
parameter_values_for_fully_qualified_parameter_names
== expected_parameter_values_for_fully_qualified_parameter_names
)
def _build_parameters(
self,
parameter_container: ParameterContainer,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
):
"""
Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and optional
details.
:return: ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and
ptional details
The algorithm operates according to the following steps:
1. Obtain batch IDs of interest using DataContext and BatchRequest (unless passed explicitly as argument). Note
that this specific BatchRequest was specified as part of configuration for the present ParameterBuilder class.
2. Set up metric_domain_kwargs and metric_value_kwargs (using configuration and/or variables and parameters).
3. Instantiate the Validator object corresponding to BatchRequest (with a temporary expectation_suite_name) in
order to have access to all Batch objects, on each of which the specified metric_name will be computed.
4. Perform metric computations and obtain the result in the array-like form (one metric value per each Batch).
5. Using the configured directives and heuristics, determine whether or not the ranges should be clipped.
6. Using the configured directives and heuristics, determine if return values should be rounded to an integer.
7. Convert the multi-dimensional metric computation results to a numpy array (for further computations).
Steps 8 -- 10 are for the "oneshot" sampling method only (the "bootstrap" method achieves same automatically):
8. Compute the mean and the standard deviation of the metric (aggregated over all the gathered Batch objects).
9. Compute number of standard deviations (as floating point) needed (around the mean) to achieve the specified
false_positive_rate (note that false_positive_rate of 0.0 would result in infinite number of standard deviations,
hence it is "nudged" by small quantity "epsilon" above 0.0 if false_positive_rate of 0.0 appears as argument).
(Please refer to "https://en.wikipedia.org/wiki/Normal_distribution" and references therein for background.)
10. Compute the "band" around the mean as the min_value and max_value (to be used in ExpectationConfiguration).
11. Return [low, high] for the desired metric as estimated by the specified sampling method.
12. Set up the arguments and call build_parameter_container() to store the parameter as part of "rule state".
"""
metric_computation_result: MetricComputationResult = self.get_metrics(
metric_name=self.metric_name,
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=self.metric_value_kwargs,
enforce_numeric_metric=self.enforce_numeric_metric,
replace_nan_with_zero=self.replace_nan_with_zero,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values: np.ndarray = metric_computation_result.metric_values
details: MetricComputationDetails = metric_computation_result.details
# Obtain sampling_method directive from "rule state" (i.e., variables and parameters); from instance variable otherwise.
sampling_method: str = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.sampling_method,
expected_return_type=str,
variables=variables,
parameters=parameters,
)
if (
sampling_method
not in NumericMetricRangeMultiBatchParameterBuilder.RECOGNIZED_SAMPLING_METHOD_NAMES
):
raise ge_exceptions.ProfilerExecutionError(
message=f"""The directive "sampling_method" for {self.__class__.__name__} can be only one of
{NumericMetricRangeMultiBatchParameterBuilder.RECOGNIZED_SAMPLING_METHOD_NAMES} ("{sampling_method}" was detected).
"""
)
estimator: Callable
etimator_kwargs: dict
if sampling_method == "bootstrap":
estimator = self._get_bootstrap_estimate
estimator_kwargs = {
"false_positive_rate": self.false_positive_rate,
"num_bootstrap_samples": self.num_bootstrap_samples,
}
else:
estimator = self._get_deterministic_estimate
estimator_kwargs = {
"false_positive_rate": self.false_positive_rate,
}
metric_value_range: np.ndarray = self._estimate_metric_value_range(
metric_values=metric_values,
estimator=estimator,
domain=domain,
variables=variables,
parameters=parameters,
**estimator_kwargs,
)
parameter_values: Dict[str, Any] = {
f"$parameter.{self.name}": {
"value": {
"value_range": metric_value_range,
},
"details": details,
},
}
build_parameter_container(
parameter_container=parameter_container, parameter_values=parameter_values
)
def build_parameters(
self,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
parameter_computation_impl: Optional[Callable] = None,
json_serialize: Optional[bool] = None,
batch_list: Optional[List[Batch]] = None,
batch_request: Optional[Union[BatchRequestBase, dict]] = None,
recompute_existing_parameter_values: bool = False,
) -> None:
"""
Args:
domain: Domain object that is context for execution of this ParameterBuilder object.
variables: attribute name/value pairs
parameters: Dictionary of ParameterContainer objects corresponding to all Domain objects in memory.
parameter_computation_impl: Object containing desired ParameterBuilder implementation.
json_serialize: If absent, use property value (in standard way, supporting variables look-up).
batch_list: Explicit list of Batch objects to supply data at runtime.
batch_request: Explicit batch_request used to supply data at runtime.
recompute_existing_parameter_values: If "True", recompute value if "fully_qualified_parameter_name" exists.
"""
fully_qualified_parameter_names: List[
str] = get_fully_qualified_parameter_names(
domain=domain,
variables=variables,
parameters=parameters,
)
if (recompute_existing_parameter_values
or self.fully_qualified_parameter_name
not in fully_qualified_parameter_names):
self.set_batch_list_or_batch_request(
batch_list=batch_list,
batch_request=batch_request,
)
resolve_evaluation_dependencies(
parameter_builder=self,
domain=domain,
variables=variables,
parameters=parameters,
fully_qualified_parameter_names=fully_qualified_parameter_names,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
if parameter_computation_impl is None:
parameter_computation_impl = self._build_parameters
parameter_computation_result: Attributes = parameter_computation_impl(
domain=domain,
variables=variables,
parameters=parameters,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
if json_serialize is None:
# Obtain json_serialize directive from "rule state" (i.e., variables and parameters); from instance variable otherwise.
json_serialize = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.json_serialize,
expected_return_type=bool,
variables=variables,
parameters=parameters,
)
parameter_values: Dict[str, Any] = {
self.fully_qualified_parameter_name:
convert_to_json_serializable(data=parameter_computation_result)
if json_serialize else parameter_computation_result,
}
build_parameter_container(
parameter_container=parameters[domain.id],
parameter_values=parameter_values,
)
def build_parameters(
self,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
parameter_computation_impl: Optional[Callable] = None,
batch_list: Optional[List[Batch]] = None,
batch_request: Optional[Union[BatchRequestBase, dict]] = None,
recompute_existing_parameter_values: bool = False,
) -> None:
"""
Args:
domain: Domain object that is context for execution of this ParameterBuilder object.
variables: attribute name/value pairs
parameters: Dictionary of ParameterContainer objects corresponding to all Domain objects in memory.
parameter_computation_impl: Object containing desired ParameterBuilder implementation.
batch_list: Explicit list of Batch objects to supply data at runtime.
batch_request: Explicit batch_request used to supply data at runtime.
recompute_existing_parameter_values: If "True", recompute value if "fully_qualified_parameter_name" exists.
"""
fully_qualified_parameter_names: List[
str
] = get_fully_qualified_parameter_names(
domain=domain,
variables=variables,
parameters=parameters,
)
if (
recompute_existing_parameter_values
or self.raw_fully_qualified_parameter_name
not in fully_qualified_parameter_names
or self.json_serialized_fully_qualified_parameter_name
not in fully_qualified_parameter_names
):
self.set_batch_list_or_batch_request(
batch_list=batch_list,
batch_request=batch_request,
)
resolve_evaluation_dependencies(
parameter_builder=self,
domain=domain,
variables=variables,
parameters=parameters,
fully_qualified_parameter_names=fully_qualified_parameter_names,
recompute_existing_parameter_values=recompute_existing_parameter_values,
)
if parameter_computation_impl is None:
parameter_computation_impl = self._build_parameters
parameter_computation_result: Attributes = parameter_computation_impl(
domain=domain,
variables=variables,
parameters=parameters,
recompute_existing_parameter_values=recompute_existing_parameter_values,
)
parameter_values: Dict[str, Any] = {
self.raw_fully_qualified_parameter_name: parameter_computation_result,
self.json_serialized_fully_qualified_parameter_name: convert_to_json_serializable(
data=parameter_computation_result
),
}
build_parameter_container(
parameter_container=parameters[domain.id],
parameter_values=parameter_values,
)
def _build_parameters(
self,
parameter_container: ParameterContainer,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
) -> ParameterContainer:
"""
Check the percentage of values matching the REGEX string, and return the best fit, or None if no
string exceeds the configured threshold.
:return: ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and optional details
"""
metric_computation_result: MetricComputationResult
metric_values: np.ndarray
metric_computation_result: MetricComputationResult = self.get_metrics(
metric_name="column_values.nonnull.count",
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=self.metric_value_kwargs,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values = metric_computation_result.metric_values
# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
metric_values = metric_values[:, 0]
nonnull_count: int = sum(metric_values)
regex_string_success_ratios: dict = {}
# Obtain candidate_regexes from "rule state" (i.e, variables and parameters); from instance variable otherwise.
candidate_regexes: Union[
Set[str], List[str],
"RegexPatternStringParameterBuilder.CANDIDATE_REGEX", # noqa: F821
] = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.candidate_regexes,
expected_return_type=None,
variables=variables,
parameters=parameters,
)
if candidate_regexes is not None and isinstance(
candidate_regexes, list):
candidate_regexes = set(candidate_regexes)
else:
candidate_regexes = RegexPatternStringParameterBuilder.CANDIDATE_REGEX
regex_string: str
match_regex_metric_value_kwargs: dict
for regex_string in candidate_regexes:
if self.metric_value_kwargs:
match_regex_metric_value_kwargs: dict = {
**self._metric_value_kwargs,
**{
"regex": regex_string
},
}
else:
match_regex_metric_value_kwargs: dict = {"regex": regex_string}
metric_computation_result: MetricComputationResult = self.get_metrics(
metric_name="column_values.match_regex.unexpected_count",
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=match_regex_metric_value_kwargs,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values = metric_computation_result.metric_values
# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
metric_values = metric_values[:, 0]
match_regex_unexpected_count: int = sum(metric_values)
success_ratio: float = (
nonnull_count - match_regex_unexpected_count) / nonnull_count
regex_string_success_ratios[regex_string] = success_ratio
# Obtain threshold from "rule state" (i.e., variables and parameters); from instance variable otherwise.
threshold: float = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self._threshold,
expected_return_type=float,
variables=variables,
parameters=parameters,
)
# get list of regex_strings that match greater than threshold
regex_string_success_list: List[
str] = self._get_regex_matched_greater_than_threshold(
regex_string_success_ratios, threshold)
# sorted regex and ratios for all evaluated candidates
sorted_ratio_list, sorted_regex_string_list = self._get_sorted_regex_and_ratios(
regex_string_success_ratios)
parameter_values: Dict[str, Any] = {
f"$parameter.{self.name}": {
"value": regex_string_success_list,
"details": {
"evaluated_regexes":
dict(zip(sorted_regex_string_list, sorted_ratio_list)),
"threshold":
threshold,
},
},
}
build_parameter_container(parameter_container=parameter_container,
parameter_values=parameter_values)
return parameter_container
def _build_parameters(
self,
parameter_container: ParameterContainer,
domain: Domain,
variables: Optional[ParameterContainer] = None,
parameters: Optional[Dict[str, ParameterContainer]] = None,
) -> ParameterContainer:
"""
Check the percentage of values matching each string, and return the best fit, or None if no
string exceeds the configured threshold.
:return: ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and optional details
"""
metric_computation_result: MetricComputationResult
metric_values: np.ndarray
metric_computation_result = self.get_metrics(
metric_name="column_values.nonnull.count",
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=self.metric_value_kwargs,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values = metric_computation_result.metric_values
# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
metric_values = metric_values[:, 0]
nonnull_count: int = sum(metric_values)
format_string_success_ratios: dict = {}
# Obtain candidate_strings from "rule state" (i.e., variables and parameters); from instance variable otherwise.
candidate_strings: Union[
Set[str],
List[str],
"SimpleDateFormatStringParameterBuilder.CANDIDATE_STRINGS", # noqa: F821
] = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.candidate_strings,
expected_return_type=None,
variables=variables,
parameters=parameters,
)
if candidate_strings is not None and isinstance(candidate_strings, list):
candidate_strings = set(candidate_strings)
else:
candidate_strings = SimpleDateFormatStringParameterBuilder.CANDIDATE_STRINGS
fmt_string: str
match_strftime_metric_value_kwargs: dict
for fmt_string in candidate_strings:
if self.metric_value_kwargs:
match_strftime_metric_value_kwargs: dict = {
**self.metric_value_kwargs,
**{"strftime_format": fmt_string},
}
else:
match_strftime_metric_value_kwargs: dict = {
"strftime_format": fmt_string,
}
metric_computation_result: MetricComputationResult = self.get_metrics(
metric_name="column_values.match_strftime_format.unexpected_count",
metric_domain_kwargs=self.metric_domain_kwargs,
metric_value_kwargs=match_strftime_metric_value_kwargs,
domain=domain,
variables=variables,
parameters=parameters,
)
metric_values = metric_computation_result.metric_values
# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
metric_values = metric_values[:, 0]
match_strftime_unexpected_count: int = sum(metric_values)
success_ratio: float = (
nonnull_count - match_strftime_unexpected_count
) / nonnull_count
format_string_success_ratios[fmt_string] = success_ratio
best_fmt_string: Optional[str] = None
best_ratio: float = 0.0
# Obtain threshold from "rule state" (i.e., variables and parameters); from instance variable otherwise.
threshold: float = get_parameter_value_and_validate_return_type(
domain=domain,
parameter_reference=self.threshold,
expected_return_type=float,
variables=variables,
parameters=parameters,
)
fmt_string: str
ratio: float
for fmt_string, ratio in format_string_success_ratios.items():
if ratio > best_ratio and ratio >= threshold:
best_fmt_string = fmt_string
best_ratio = ratio
parameter_values: Dict[str, Any] = {
f"$parameter.{self.name}": {
"value": best_fmt_string,
"details": {"success_ratio": best_ratio},
},
}
build_parameter_container(
parameter_container=parameter_container, parameter_values=parameter_values
)
return parameter_container
