class RuleConfigSchema(NotNullSchema):
class Meta:
unknown = INCLUDE
__config_class__ = RuleConfig
domain_builder = fields.Nested(
DomainBuilderConfigSchema,
required=False,
allow_none=True,
)
parameter_builders = fields.List(
cls_or_instance=fields.Nested(
ParameterBuilderConfigSchema,
required=True,
allow_none=False,
),
required=False,
allow_none=True,
)
expectation_configuration_builders = fields.List(
cls_or_instance=fields.Nested(
ExpectationConfigurationBuilderConfigSchema,
required=True,
allow_none=False,
),
required=True,
allow_none=False,
)
class ParameterBuilderConfigSchema(NotNullSchema):
class Meta:
unknown = INCLUDE
__config_class__ = ParameterBuilderConfig
name = fields.String(
required=True,
allow_none=False,
)
module_name = fields.String(
required=False,
allow_none=True,
missing="great_expectations.rule_based_profiler.parameter_builder",
)
class_name = fields.String(
required=True,
allow_none=False,
)
evaluation_parameter_builder_configs = fields.List(
cls_or_instance=fields.Nested(
lambda: ParameterBuilderConfigSchema(),
required=True,
allow_none=False,
),
required=False,
allow_none=True,
)
class ExpectationSuiteValidationResultSchema(Schema):
success = fields.Bool()
results = fields.List(fields.Nested(ExpectationValidationResultSchema))
evaluation_parameters = fields.Dict()
statistics = fields.Dict()
meta = fields.Dict(allow_none=True)
ge_cloud_id = fields.UUID(required=False, allow_none=True)
# noinspection PyUnusedLocal
@pre_dump
def prepare_dump(self, data, **kwargs):
data = deepcopy(data)
if isinstance(data, ExpectationSuiteValidationResult):
data.meta = convert_to_json_serializable(data=data.meta)
data.statistics = convert_to_json_serializable(
data=data.statistics)
elif isinstance(data, dict):
data["meta"] = convert_to_json_serializable(data=data.get("meta"))
data["statistics"] = convert_to_json_serializable(
data=data.get("statistics"))
return data
# noinspection PyUnusedLocal
@post_load
def make_expectation_suite_validation_result(self, data, **kwargs):
return ExpectationSuiteValidationResult(**data)
class ExpectationSuiteSchema(Schema):
expectation_suite_name = fields.Str()
expectations = fields.List(fields.Nested(ExpectationConfigurationSchema))
evaluation_parameters = fields.Dict(allow_none=True)
data_asset_type = fields.Str(allow_none=True)
meta = fields.Dict()
# NOTE: 20191107 - JPC - we may want to remove clean_empty and update tests to require the other fields;
# doing so could also allow us not to have to make a copy of data in the pre_dump method.
def clean_empty(self, data):
if not hasattr(data, "evaluation_parameters"):
pass
elif len(data.evaluation_parameters) == 0:
del data.evaluation_parameters
if not hasattr(data, "meta"):
pass
elif data.meta is None or data.meta == []:
pass
elif len(data.meta) == 0:
del data.meta
return data
# noinspection PyUnusedLocal
@pre_dump
def prepare_dump(self, data, **kwargs):
data = deepcopy(data)
data.meta = convert_to_json_serializable(data.meta)
data = self.clean_empty(data)
return data
# noinspection PyUnusedLocal
@post_load
def make_expectation_suite(self, data, **kwargs):
return ExpectationSuite(**data)
class RenderedAtomicValueSchema(Schema):
class Meta:
unknown = INCLUDE
schema = fields.Dict(required=False, allow_none=True)
header = fields.Dict(required=False, allow_none=True)
# for StringValueType
template = fields.String(required=False, allow_none=True)
params = fields.Dict(required=False, allow_none=True)
# for TableType
header_row = fields.List(fields.Dict, required=False, allow_none=True)
table = fields.List(
fields.List(fields.Dict, required=False, allow_none=True))
# for GraphType
graph = fields.Dict(required=False, allow_none=True)
# for UnknownType
kwargs = fields.Dict(required=False, allow_none=True)
@post_load
def create_value_obj(self, data, **kwargs):
return RenderedAtomicValue(**data)
REMOVE_KEYS_IF_NONE = [
"template",
"table",
"params",
"header_row",
"table",
"graph",
"kwargs",
]
@post_dump
def clean_null_attrs(self, data: dict, **kwargs: dict) -> dict:
"""Removes the attributes in RenderedAtomicValueSchema.REMOVE_KEYS_IF_NONE during serialization if
their values are None."""
data = deepcopy(data)
for key in RenderedAtomicValueSchema.REMOVE_KEYS_IF_NONE:
if key == "graph" and key in data and data[key].graph is None:
data.pop(key)
elif key in data and data[key] is None:
data.pop(key)
return data
class RenderedAtomicValueSchema(Schema):
class Meta:
unknown = INCLUDE
# for StringType
template = fields.String(required=False, allow_none=True)
params = fields.Dict(required=False, allow_none=True)
schema = fields.Dict(required=False, allow_none=True)
# for TableType
header = fields.Dict(required=False, allow_none=True)
header_row = fields.List(fields.Dict, required=False, allow_none=True)
table = fields.List(fields.List(fields.Dict, required=False, allow_none=True))
# for GraphType
graph = fields.String(required=False, allow_none=True)
@post_load
def create_value_obj(self, data, **kwargs):
return RenderedAtomicValue(**data)
class ExpectationSuiteSchema(Schema):
expectation_suite_name = fields.Str()
ge_cloud_id = fields.UUID(required=False, allow_none=True)
expectations = fields.List(fields.Nested(ExpectationConfigurationSchema))
evaluation_parameters = fields.Dict(allow_none=True)
data_asset_type = fields.Str(allow_none=True)
meta = fields.Dict()
# NOTE: 20191107 - JPC - we may want to remove clean_empty and update tests to require the other fields;
# doing so could also allow us not to have to make a copy of data in the pre_dump method.
# noinspection PyMethodMayBeStatic
def clean_empty(self, data):
if isinstance(data, ExpectationSuite):
if not hasattr(data, "evaluation_parameters"):
pass
elif len(data.evaluation_parameters) == 0:
del data.evaluation_parameters
if not hasattr(data, "meta"):
pass
elif data.meta is None or data.meta == []:
pass
elif len(data.meta) == 0:
del data.meta
elif isinstance(data, dict):
if not data.get("evaluation_parameters"):
pass
elif len(data.get("evaluation_parameters")) == 0:
data.pop("evaluation_parameters")
if not data.get("meta"):
pass
elif data.get("meta") is None or data.get("meta") == []:
pass
elif len(data.get("meta")) == 0:
data.pop("meta")
return data
# noinspection PyUnusedLocal
@pre_dump
def prepare_dump(self, data, **kwargs):
data = deepcopy(data)
if isinstance(data, ExpectationSuite):
data.meta = convert_to_json_serializable(data.meta)
elif isinstance(data, dict):
data["meta"] = convert_to_json_serializable(data.get("meta"))
data = self.clean_empty(data)
return data
class ExpectationSuiteValidationResultSchema(Schema):
success = fields.Bool()
results = fields.List(fields.Nested(ExpectationValidationResultSchema))
evaluation_parameters = fields.Dict()
statistics = fields.Dict()
meta = fields.Dict(allow_none=True)
# noinspection PyUnusedLocal
@pre_dump
def prepare_dump(self, data, **kwargs):
data = deepcopy(data)
data.meta = convert_to_json_serializable(data.meta)
return data
# noinspection PyUnusedLocal
@post_load
def make_expectation_suite_validation_result(self, data, **kwargs):
return ExpectationSuiteValidationResult(**data)
class AssetConfigSchema(Schema):
class Meta:
unknown = INCLUDE
base_directory = fields.String(required=False, allow_none=True)
glob_directive = fields.String(required=False, allow_none=True)
pattern = fields.String(required=False, allow_none=True)
group_names = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
@validates_schema
def validate_schema(self, data, **kwargs):
pass
# noinspection PyUnusedLocal
@post_load
def make_asset_config(self, data, **kwargs):
return AssetConfig(**data)
class ExpectationConfigurationBuilderConfigSchema(NotNullSchema):
class Meta:
unknown = INCLUDE
__config_class__ = ExpectationConfigurationBuilderConfig
module_name = fields.String(
required=False,
allow_none=True,
missing=
"great_expectations.rule_based_profiler.expectation_configuration_builder",
)
class_name = fields.String(
required=True,
allow_none=False,
)
expectation_type = fields.Str(
required=True,
error_messages={
"required":
"expectation_type missing in expectation configuration builder"
},
)
meta = fields.Dict(
keys=fields.String(
required=True,
allow_none=False,
),
required=False,
allow_none=True,
)
validation_parameter_builder_configs = fields.List(
cls_or_instance=fields.Nested(
lambda: ParameterBuilderConfigSchema(),
required=True,
allow_none=False,
),
required=False,
allow_none=True,
)
class ExpectationValidationResultSchema(Schema):
success = fields.Bool(required=False, allow_none=True)
expectation_config = fields.Nested(lambda: ExpectationConfigurationSchema,
required=False,
allow_none=True)
result = fields.Dict(required=False, allow_none=True)
meta = fields.Dict(required=False, allow_none=True)
exception_info = fields.Dict(required=False, allow_none=True)
rendered_content = fields.List(
fields.Nested(lambda: RenderedAtomicContentSchema,
required=False,
allow_none=True))
# noinspection PyUnusedLocal
@pre_dump
def convert_result_to_serializable(self, data, **kwargs):
data = deepcopy(data)
if isinstance(data, ExpectationValidationResult):
data.result = convert_to_json_serializable(data.result)
elif isinstance(data, dict):
data["result"] = convert_to_json_serializable(data.get("result"))
return data
REMOVE_KEYS_IF_NONE = ["rendered_content"]
@post_dump
def clean_null_attrs(self, data: dict, **kwargs: dict) -> dict:
"""Removes the attributes in ExpectationValidationResultSchema.REMOVE_KEYS_IF_NONE during serialization if
their values are None."""
data = deepcopy(data)
for key in ExpectationConfigurationSchema.REMOVE_KEYS_IF_NONE:
if key in data and data[key] is None:
data.pop(key)
return data
# noinspection PyUnusedLocal
@post_load
def make_expectation_validation_result(self, data, **kwargs):
return ExpectationValidationResult(**data)
class DataConnectorConfigSchema(Schema):
class Meta:
unknown = INCLUDE
class_name = fields.String(required=True)
module_name = fields.String(
missing="great_expectations.datasource.data_connector")
assets = fields.Dict(
keys=fields.Str(),
values=fields.Nested(AssetConfigSchema,
required=False,
allow_none=True),
required=False,
allow_none=True,
)
base_directory = fields.String(required=False, allow_none=True)
glob_directive = fields.String(required=False, allow_none=True)
default_regex = fields.Dict(required=False, allow_none=True)
runtime_keys = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
bucket = fields.String(required=False, allow_none=True)
prefix = fields.String(required=False, allow_none=True)
delimiter = fields.String(required=False, allow_none=True)
max_keys = fields.Integer(required=False, allow_none=True)
boto3_options = fields.Dict(keys=fields.Str(),
values=fields.Str(),
required=False,
allow_none=True)
data_asset_name_prefix = fields.String(required=False, allow_none=True)
data_asset_name_suffix = fields.String(required=False, allow_none=True)
include_schema_name = fields.Boolean(required=False, allow_none=True)
splitter_method = fields.String(required=False, allow_none=True)
splitter_kwargs = fields.Dict(required=False, allow_none=True)
sampling_method = fields.String(required=False, allow_none=True)
sampling_kwargs = fields.Dict(required=False, allow_none=True)
excluded_tables = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
included_tables = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
skip_inapplicable_tables = fields.Boolean(required=False, allow_none=True)
@validates_schema
def validate_schema(self, data, **kwargs):
# If a class_name begins with the dollar sign ("$"), then it is assumed to be a variable name to be substituted.
if data["class_name"][0] == "$":
return
if ("default_regex" in data) and not (data["class_name"] in [
"InferredAssetFilesystemDataConnector",
"ConfiguredAssetFilesystemDataConnector",
"InferredAssetS3DataConnector",
"ConfiguredAssetS3DataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by a
subclass of the FilePathDataConnector class (your data conntector is "{data['class_name']}"). Please update your
configuration to continue.
""")
if ("glob_directive" in data) and not (data["class_name"] in [
"InferredAssetFilesystemDataConnector",
"ConfiguredAssetFilesystemDataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by a
filesystem type of the data connector (your data conntector is "{data['class_name']}"). Please update your
configuration to continue.
""")
if ("bucket" in data or "prefix" in data or "delimiter" in data
or "max_keys" in data) and not (data["class_name"] in [
"InferredAssetS3DataConnector",
"ConfiguredAssetS3DataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by an
S3 type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to
continue.
""")
if ("data_asset_name_prefix" in data
or "data_asset_name_suffix" in data
or "include_schema_name" in data or "splitter_method" in data
or "splitter_kwargs" in data or "sampling_method" in data
or "sampling_kwargs" in data or "excluded_tables" in data
or "included_tables" in data or "skip_inapplicable_tables"
in data) and not (data["class_name"] in [
"InferredAssetSqlDataConnector",
"ConfiguredAssetSqlDataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by an
SQL type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to
continue.
""")
# noinspection PyUnusedLocal
@post_load
def make_data_connector_config(self, data, **kwargs):
return DataConnectorConfig(**data)
