def add_domain(
self,
domain: Domain,
allow_duplicates: bool = False,
) -> None:
domain_cursor: Domain
if not allow_duplicates and domain.id in [
domain_cursor.id for domain_cursor in self.domains
]:
raise ge_exceptions.ProfilerConfigurationError(
f"""Error: Domain\n{domain}\nalready exists. In order to add it, either pass "allow_duplicates=True" \
or call "RuleState.remove_domain_if_exists()" with Domain having ID equal to "{domain.id}" as argument first.
""")
self.domains.append(domain)
def _validate_builder_override_config(builder_config: dict):
"""
In order to insure successful instantiation of custom builder classes using "instantiate_class_from_config()",
candidate builder override configurations are required to supply both "class_name" and "module_name" attributes.
:param builder_config: candidate builder override configuration
:raises: ProfilerConfigurationError
"""
if not all([
isinstance(builder_config, dict),
"class_name" in builder_config,
"module_name" in builder_config,
]):
raise ge_exceptions.ProfilerConfigurationError(
'Both "class_name" and "module_name" must be specified.')
def initialize_parameter_container_for_domain(
self,
domain: Domain,
overwrite: bool = True,
) -> None:
if not overwrite and domain.id in self.parameters:
raise ge_exceptions.ProfilerConfigurationError(
f"""Error: ParameterContainer for Domain\n{domain}\nalready exists. In order to overwrite it, either \
pass "overwrite=True" or call "RuleState.remove_parameter_container_from_domain()" with Domain having ID equal to \
"{domain.id}" as argument first.
""")
parameter_container: ParameterContainer = ParameterContainer(
parameter_nodes=None)
self._parameters[domain.id] = parameter_container
def set_batch_data(
self,
batch_list: Optional[List[Batch]] = None,
batch_request: Optional[Union[BatchRequestBase, dict]] = None,
) -> None:
arg: Any
num_supplied_batch_specification_args: int = sum(
[0 if arg is None else 1 for arg in (
batch_list,
batch_request,
)])
if num_supplied_batch_specification_args > 1:
raise ge_exceptions.ProfilerConfigurationError(
f'Please pass at most one of "batch_list" and "batch_request" arguments (you passed {num_supplied_batch_specification_args} arguments).'
)
if batch_list is None:
self.batch_request = batch_request
else:
self.batch_list = batch_list
def _init_rule(
self,
rule_name: str,
rule_config: Dict[str, Any],
) -> Rule:
# Config is validated through schema but do a sanity check
attr: str
for attr in (
"domain_builder",
"expectation_configuration_builders",
):
if attr not in rule_config:
raise ge_exceptions.ProfilerConfigurationError(
message=
f'Invalid rule "{rule_name}": missing mandatory {attr}.')
# Instantiate builder attributes
domain_builder: DomainBuilder = RuleBasedProfiler._init_rule_domain_builder(
domain_builder_config=rule_config["domain_builder"],
data_context=self._data_context,
)
parameter_builders: Optional[List[
ParameterBuilder]] = RuleBasedProfiler._init_rule_parameter_builders(
parameter_builder_configs=rule_config.get(
"parameter_builders"),
data_context=self._data_context,
)
expectation_configuration_builders: List[
ExpectationConfigurationBuilder] = RuleBasedProfiler._init_rule_expectation_configuration_builders(
expectation_configuration_builder_configs=rule_config[
"expectation_configuration_builders"])
# Compile previous steps and package into a Rule object
return Rule(
name=rule_name,
domain_builder=domain_builder,
parameter_builders=parameter_builders,
expectation_configuration_builders=
expectation_configuration_builders,
)
def __init__(
self,
*,
profiler_config: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None,
data_context: Optional[DataContext] = None,
):
"""
Create a new Profiler using configured rules.
For a rule or an item in a rule configuration, instantiates the following if
available: a domain builder, a parameter builder, and a configuration builder.
These will be used to define profiler computation patterns.
Args:
profiler_config: Variables and Rules configuration as a dictionary
data_context: DataContext object that defines a full runtime environment (data access, etc.)
"""
self._profiler_config = profiler_config
self._data_context = data_context
self._rules = []
rules_configs: Dict[str, Dict[str, Any]] = self._profiler_config.get(
"rules", {})
rule_name: str
rule_config: Dict[str, Any]
for rule_name, rule_config in rules_configs.items():
domain_builder_config: dict = rule_config.get("domain_builder")
if domain_builder_config is None:
raise ge_exceptions.ProfilerConfigurationError(
message=
f'Invalid rule "{rule_name}": no domain_builder found.')
domain_builder: DomainBuilder = instantiate_class_from_config(
config=domain_builder_config,
runtime_environment={"data_context": data_context},
config_defaults={
"module_name":
"great_expectations.rule_based_profiler.domain_builder"
},
)
parameter_builders: List[ParameterBuilder] = []
parameter_builder_configs: dict = rule_config.get(
"parameter_builders")
if parameter_builder_configs:
parameter_builder_config: dict
for parameter_builder_config in parameter_builder_configs:
parameter_builders.append(
instantiate_class_from_config(
config=parameter_builder_config,
runtime_environment={"data_context": data_context},
config_defaults={
"module_name":
"great_expectations.rule_based_profiler.parameter_builder"
},
))
expectation_configuration_builders: List[
ExpectationConfigurationBuilder] = []
expectation_configuration_builder_configs: dict = rule_config.get(
"expectation_configuration_builders")
if expectation_configuration_builder_configs:
expectation_configuration_builder_config: dict
for (expectation_configuration_builder_config
) in expectation_configuration_builder_configs:
expectation_configuration_builders.append(
instantiate_class_from_config(
config=expectation_configuration_builder_config,
runtime_environment={},
config_defaults={
"class_name":
"DefaultExpectationConfigurationBuilder",
"module_name":
"great_expectations.rule_based_profiler.expectation_configuration_builder",
},
))
variables_configs: Dict[str, Dict] = self._profiler_config.get(
"variables", {})
variables: Optional[ParameterContainer] = None
if variables_configs:
variables = build_parameter_container_for_variables(
variables_configs=variables_configs)
self._rules.append(
Rule(
name=rule_name,
domain_builder=domain_builder,
parameter_builders=parameter_builders,
expectation_configuration_builders=
expectation_configuration_builders,
variables=variables,
))