def reconcile_rule_variables(
variables: ParameterContainer,
variables_config: dict,
reconciliation_strategy:
ReconciliationStrategy = DEFAULT_RECONCILATION_DIRECTIVES.variables,
) -> dict:
"""
Rule "variables" reconciliation involves combining the variables, instantiated from Rule configuration
(e.g., stored in a YAML file managed by the Profiler store), with the variables override, possibly supplied
as part of the candiate override rule configuration.
The reconciliation logic for "variables" is of the "replace" nature: An override value complements the
original on key "miss", and replaces the original on key "hit" (or "collision"), because "variables" is a
unique member for a Rule.
:param variables: existing variables of a Rule
:param variables_config: variables configuration override, supplied in dictionary (configuration) form
:param reconciliation_strategy: one of update, nested_update, or overwrite ways of reconciling overwrites
:return: reconciled variables configuration, returned in dictionary (configuration) form
"""
effective_variables_config: dict = convert_variables_to_dict(
variables=variables)
if variables_config:
if reconciliation_strategy == ReconciliationStrategy.NESTED_UPDATE:
effective_variables_config = nested_update(
effective_variables_config,
variables_config,
)
elif reconciliation_strategy == ReconciliationStrategy.REPLACE:
effective_variables_config = variables_config
elif reconciliation_strategy == ReconciliationStrategy.UPDATE:
effective_variables_config.update(variables_config)
return effective_variables_config
def resolve_config_using_acceptable_arguments(
cls,
profiler: "RuleBasedProfiler", # noqa: F821
variables: Optional[Dict[str, Any]] = None,
rules: Optional[Dict[str, Dict[str, Any]]] = None,
) -> "RuleBasedProfilerConfig": # noqa: F821
"""Reconciles variables/rules by taking into account runtime overrides and variable substitution.
Utilized in usage statistics to interact with the args provided in `RuleBasedProfiler.run()`.
NOTE: This is a lightweight version of the RBP's true reconiliation logic - see
`reconcile_profiler_variables` and `reconcile_profiler_rules`.
Args:
profiler: The profiler used to invoke `run()`.
variables: Any runtime override variables.
rules: Any runtime override rules.
Returns:
An instance of RuleBasedProfilerConfig that represents the reconciled profiler.
"""
effective_variables: Optional[
ParameterContainer] = profiler.reconcile_profiler_variables(
variables=variables, )
runtime_variables: Optional[Dict[str,
Any]] = convert_variables_to_dict(
variables=effective_variables)
effective_rules: List[
"Rule"] = profiler.reconcile_profiler_rules( # noqa: F821
rules=rules, )
rule: "Rule" # noqa: F821
effective_rules_dict: Dict[str, "Rule"] = { # noqa: F821
rule.name: rule
for rule in effective_rules
}
runtime_rules: Dict[str, dict] = {
name: RuleBasedProfilerConfig._substitute_variables_in_config(
rule=rule,
variables_container=effective_variables,
)
for name, rule in effective_rules_dict.items()
}
return cls(
name=profiler.config.name,
config_version=profiler.config.config_version,
variables=runtime_variables,
rules=runtime_rules,
)
def to_json_dict(self) -> dict:
"""
# TODO: <Alex>2/4/2022</Alex>
This implementation of "SerializableDictDot.to_json_dict() occurs frequently and should ideally serve as the
reference implementation in the "SerializableDictDot" class itself. However, the circular import dependencies,
due to the location of the "great_expectations/types/__init__.py" and "great_expectations/core/util.py" modules
make this refactoring infeasible at the present time.
"""
dict_obj: dict = self.to_dict()
variables_dict: Optional[Dict[str, Any]] = convert_variables_to_dict(
variables=self.variables)
dict_obj["variables"] = variables_dict
serializeable_dict: dict = convert_to_json_serializable(data=dict_obj)
return serializeable_dict
def run(
self,
variables: Optional[ParameterContainer] = None,
batch_list: Optional[List[Batch]] = None,
batch_request: Optional[Union[BatchRequestBase, dict]] = None,
recompute_existing_parameter_values: bool = False,
reconciliation_directives:
ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES,
rule_state: Optional[RuleState] = None,
) -> RuleState:
"""
Builds a list of Expectation Configurations, returning a single Expectation Configuration entry for every
ConfigurationBuilder available based on the instantiation.
Args:
variables: Attribute name/value pairs, commonly-used in Builder objects
batch_list: Explicit list of Batch objects to supply data at runtime
batch_request: Explicit batch_request used to supply data at runtime
recompute_existing_parameter_values: If "True", recompute value if "fully_qualified_parameter_name" exists
reconciliation_directives: directives for how each rule component should be overwritten
rule_state: holds "Rule" execution state and responds to "execution_time_property_name" ("execution_time")
Returns:
RuleState representing effect of executing Rule
"""
variables = build_parameter_container_for_variables(
variables_configs=reconcile_rule_variables(
variables=variables,
variables_config=convert_variables_to_dict(
variables=self.variables),
reconciliation_strategy=reconciliation_directives.variables,
))
domains: List[Domain] = ([] if self.domain_builder is None else
self.domain_builder.get_domains(
rule_name=self.name,
variables=variables,
batch_list=batch_list,
batch_request=batch_request,
))
if rule_state is None:
rule_state = RuleState()
rule_state.rule = self
rule_state.variables = variables
rule_state.domains = domains
rule_state.reset_parameter_containers()
pbar_method: Callable = determine_progress_bar_method_by_environment()
domain: Domain
for domain in pbar_method(
domains,
desc="Profiling Dataset:",
position=1,
leave=False,
bar_format="{desc:25}{percentage:3.0f}%|{bar}{r_bar}",
):
rule_state.initialize_parameter_container_for_domain(domain=domain)
parameter_builders: List[
ParameterBuilder] = self.parameter_builders or []
parameter_builder: ParameterBuilder
for parameter_builder in parameter_builders:
parameter_builder.build_parameters(
domain=domain,
variables=variables,
parameters=rule_state.parameters,
parameter_computation_impl=None,
batch_list=batch_list,
batch_request=batch_request,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
expectation_configuration_builders: List[
ExpectationConfigurationBuilder] = (
self.expectation_configuration_builders or [])
expectation_configuration_builder: ExpectationConfigurationBuilder
for expectation_configuration_builder in expectation_configuration_builders:
expectation_configuration_builder.resolve_validation_dependencies(
domain=domain,
variables=variables,
parameters=rule_state.parameters,
batch_list=batch_list,
batch_request=batch_request,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
return rule_state
def run(
self,
variables: Optional[ParameterContainer] = None,
batch_list: Optional[List[Batch]] = None,
batch_request: Optional[Union[BatchRequestBase, dict]] = None,
recompute_existing_parameter_values: bool = False,
reconciliation_directives:
ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES,
) -> RuleState:
"""
Builds a list of Expectation Configurations, returning a single Expectation Configuration entry for every
ConfigurationBuilder available based on the instantiation.
Args:
variables: Attribute name/value pairs, commonly-used in Builder objects
batch_list: Explicit list of Batch objects to supply data at runtime
batch_request: Explicit batch_request used to supply data at runtime
recompute_existing_parameter_values: If "True", recompute value if "fully_qualified_parameter_name" exists
reconciliation_directives: directives for how each rule component should be overwritten
Returns:
RuleState representing effect of executing Rule
"""
variables = build_parameter_container_for_variables(
variables_configs=reconcile_rule_variables(
variables=variables,
variables_config=convert_variables_to_dict(
variables=self.variables),
reconciliation_strategy=reconciliation_directives.variables,
))
domains: List[Domain] = ([] if self.domain_builder is None else
self.domain_builder.get_domains(
rule_name=self.name,
variables=variables,
batch_list=batch_list,
batch_request=batch_request,
))
rule_state: RuleState = RuleState(
rule=self,
variables=variables,
domains=domains,
)
rule_state.reset_parameter_containers()
domain: Domain
for domain in domains:
rule_state.initialize_parameter_container_for_domain(domain=domain)
parameter_builders: List[
ParameterBuilder] = self.parameter_builders or []
parameter_builder: ParameterBuilder
for parameter_builder in parameter_builders:
parameter_builder.build_parameters(
domain=domain,
variables=variables,
parameters=rule_state.parameters,
parameter_computation_impl=None,
json_serialize=None,
batch_list=batch_list,
batch_request=batch_request,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
expectation_configuration_builders: List[
ExpectationConfigurationBuilder] = (
self.expectation_configuration_builders or [])
expectation_configuration_builder: ExpectationConfigurationBuilder
for expectation_configuration_builder in expectation_configuration_builders:
expectation_configuration_builder.resolve_validation_dependencies(
domain=domain,
variables=variables,
parameters=rule_state.parameters,
batch_list=batch_list,
batch_request=batch_request,
recompute_existing_parameter_values=
recompute_existing_parameter_values,
)
return rule_state
def _build_profiler(self) -> None:
"""
Builds "RuleBasedProfiler", corresponding to present DataAssistant use case.
Starts with empty "RuleBasedProfiler" (initialized in constructor) and adds Rule objects.
Subclasses can add custom "Rule" objects as appropriate for their respective particular DataAssistant use cases.
"""
variables: dict = {}
profiler: Optional[BaseRuleBasedProfiler]
rules: List[Rule]
rule: Rule
domain_builder: DomainBuilder
parameter_builders: List[ParameterBuilder]
expectation_configuration_builders: List[ExpectationConfigurationBuilder]
"""
For each Self-Initializing "Expectation" as specified by "DataAssistant.expectation_kwargs_by_expectation_type"
interface property, retrieve its "RuleBasedProfiler" configuration, construct "Rule" object based on it, while
incorporating metrics "ParameterBuilder" objects for "MetricDomainTypes", emitted by "DomainBuilder"
of comprised "Rule", specified by "DataAssistant.metrics_parameter_builders_by_domain" interface property.
Append this "Rule" object to overall DataAssistant "RuleBasedProfiler" object; incorporate "variables" as well.
"""
expectation_type: str
expectation_kwargs: Dict[str, Any]
for (
expectation_type,
expectation_kwargs,
) in self.expectation_kwargs_by_expectation_type.items():
profiler = self._validator.build_rule_based_profiler_for_expectation(
expectation_type=expectation_type
)(**expectation_kwargs)
variables.update(convert_variables_to_dict(variables=profiler.variables))
rules = profiler.rules
for rule in rules:
domain_builder = rule.domain_builder
parameter_builders = rule.parameter_builders or []
parameter_builders.extend(
self.metrics_parameter_builders_by_domain[
Domain(
domain_builder.domain_type,
)
]
)
expectation_configuration_builders = (
rule.expectation_configuration_builders or []
)
self.profiler.add_rule(
rule=Rule(
name=rule.name,
variables=rule.variables,
domain_builder=domain_builder,
parameter_builders=parameter_builders,
expectation_configuration_builders=expectation_configuration_builders,
)
)
self.profiler.variables = self.profiler.reconcile_profiler_variables(
variables=variables,
reconciliation_strategy=DEFAULT_RECONCILATION_DIRECTIVES.variables,
)