class RuleConfigSchema(NotNullSchema): class Meta: unknown = INCLUDE __config_class__ = RuleConfig domain_builder = fields.Nested( DomainBuilderConfigSchema, required=False, allow_none=True, ) parameter_builders = fields.List( cls_or_instance=fields.Nested( ParameterBuilderConfigSchema, required=True, allow_none=False, ), required=False, allow_none=True, ) expectation_configuration_builders = fields.List( cls_or_instance=fields.Nested( ExpectationConfigurationBuilderConfigSchema, required=True, allow_none=False, ), required=True, allow_none=False, )
class ParameterBuilderConfigSchema(NotNullSchema): class Meta: unknown = INCLUDE __config_class__ = ParameterBuilderConfig name = fields.String( required=True, allow_none=False, ) module_name = fields.String( required=False, allow_none=True, missing="great_expectations.rule_based_profiler.parameter_builder", ) class_name = fields.String( required=True, allow_none=False, ) evaluation_parameter_builder_configs = fields.List( cls_or_instance=fields.Nested( lambda: ParameterBuilderConfigSchema(), required=True, allow_none=False, ), required=False, allow_none=True, )
class ExpectationSuiteValidationResultSchema(Schema): success = fields.Bool() results = fields.List(fields.Nested(ExpectationValidationResultSchema)) evaluation_parameters = fields.Dict() statistics = fields.Dict() meta = fields.Dict(allow_none=True) ge_cloud_id = fields.UUID(required=False, allow_none=True) # noinspection PyUnusedLocal @pre_dump def prepare_dump(self, data, **kwargs): data = deepcopy(data) if isinstance(data, ExpectationSuiteValidationResult): data.meta = convert_to_json_serializable(data=data.meta) data.statistics = convert_to_json_serializable( data=data.statistics) elif isinstance(data, dict): data["meta"] = convert_to_json_serializable(data=data.get("meta")) data["statistics"] = convert_to_json_serializable( data=data.get("statistics")) return data # noinspection PyUnusedLocal @post_load def make_expectation_suite_validation_result(self, data, **kwargs): return ExpectationSuiteValidationResult(**data)
class ExpectationSuiteSchema(Schema): expectation_suite_name = fields.Str() expectations = fields.List(fields.Nested(ExpectationConfigurationSchema)) evaluation_parameters = fields.Dict(allow_none=True) data_asset_type = fields.Str(allow_none=True) meta = fields.Dict() # NOTE: 20191107 - JPC - we may want to remove clean_empty and update tests to require the other fields; # doing so could also allow us not to have to make a copy of data in the pre_dump method. def clean_empty(self, data): if not hasattr(data, "evaluation_parameters"): pass elif len(data.evaluation_parameters) == 0: del data.evaluation_parameters if not hasattr(data, "meta"): pass elif data.meta is None or data.meta == []: pass elif len(data.meta) == 0: del data.meta return data # noinspection PyUnusedLocal @pre_dump def prepare_dump(self, data, **kwargs): data = deepcopy(data) data.meta = convert_to_json_serializable(data.meta) data = self.clean_empty(data) return data # noinspection PyUnusedLocal @post_load def make_expectation_suite(self, data, **kwargs): return ExpectationSuite(**data)
class RenderedAtomicValueSchema(Schema): class Meta: unknown = INCLUDE schema = fields.Dict(required=False, allow_none=True) header = fields.Dict(required=False, allow_none=True) # for StringValueType template = fields.String(required=False, allow_none=True) params = fields.Dict(required=False, allow_none=True) # for TableType header_row = fields.List(fields.Dict, required=False, allow_none=True) table = fields.List( fields.List(fields.Dict, required=False, allow_none=True)) # for GraphType graph = fields.Dict(required=False, allow_none=True) # for UnknownType kwargs = fields.Dict(required=False, allow_none=True) @post_load def create_value_obj(self, data, **kwargs): return RenderedAtomicValue(**data) REMOVE_KEYS_IF_NONE = [ "template", "table", "params", "header_row", "table", "graph", "kwargs", ] @post_dump def clean_null_attrs(self, data: dict, **kwargs: dict) -> dict: """Removes the attributes in RenderedAtomicValueSchema.REMOVE_KEYS_IF_NONE during serialization if their values are None.""" data = deepcopy(data) for key in RenderedAtomicValueSchema.REMOVE_KEYS_IF_NONE: if key == "graph" and key in data and data[key].graph is None: data.pop(key) elif key in data and data[key] is None: data.pop(key) return data
class RenderedAtomicValueSchema(Schema): class Meta: unknown = INCLUDE # for StringType template = fields.String(required=False, allow_none=True) params = fields.Dict(required=False, allow_none=True) schema = fields.Dict(required=False, allow_none=True) # for TableType header = fields.Dict(required=False, allow_none=True) header_row = fields.List(fields.Dict, required=False, allow_none=True) table = fields.List(fields.List(fields.Dict, required=False, allow_none=True)) # for GraphType graph = fields.String(required=False, allow_none=True) @post_load def create_value_obj(self, data, **kwargs): return RenderedAtomicValue(**data)
class ExpectationSuiteSchema(Schema): expectation_suite_name = fields.Str() ge_cloud_id = fields.UUID(required=False, allow_none=True) expectations = fields.List(fields.Nested(ExpectationConfigurationSchema)) evaluation_parameters = fields.Dict(allow_none=True) data_asset_type = fields.Str(allow_none=True) meta = fields.Dict() # NOTE: 20191107 - JPC - we may want to remove clean_empty and update tests to require the other fields; # doing so could also allow us not to have to make a copy of data in the pre_dump method. # noinspection PyMethodMayBeStatic def clean_empty(self, data): if isinstance(data, ExpectationSuite): if not hasattr(data, "evaluation_parameters"): pass elif len(data.evaluation_parameters) == 0: del data.evaluation_parameters if not hasattr(data, "meta"): pass elif data.meta is None or data.meta == []: pass elif len(data.meta) == 0: del data.meta elif isinstance(data, dict): if not data.get("evaluation_parameters"): pass elif len(data.get("evaluation_parameters")) == 0: data.pop("evaluation_parameters") if not data.get("meta"): pass elif data.get("meta") is None or data.get("meta") == []: pass elif len(data.get("meta")) == 0: data.pop("meta") return data # noinspection PyUnusedLocal @pre_dump def prepare_dump(self, data, **kwargs): data = deepcopy(data) if isinstance(data, ExpectationSuite): data.meta = convert_to_json_serializable(data.meta) elif isinstance(data, dict): data["meta"] = convert_to_json_serializable(data.get("meta")) data = self.clean_empty(data) return data
class ExpectationSuiteValidationResultSchema(Schema): success = fields.Bool() results = fields.List(fields.Nested(ExpectationValidationResultSchema)) evaluation_parameters = fields.Dict() statistics = fields.Dict() meta = fields.Dict(allow_none=True) # noinspection PyUnusedLocal @pre_dump def prepare_dump(self, data, **kwargs): data = deepcopy(data) data.meta = convert_to_json_serializable(data.meta) return data # noinspection PyUnusedLocal @post_load def make_expectation_suite_validation_result(self, data, **kwargs): return ExpectationSuiteValidationResult(**data)
class AssetConfigSchema(Schema): class Meta: unknown = INCLUDE base_directory = fields.String(required=False, allow_none=True) glob_directive = fields.String(required=False, allow_none=True) pattern = fields.String(required=False, allow_none=True) group_names = fields.List(cls_or_instance=fields.Str(), required=False, allow_none=True) @validates_schema def validate_schema(self, data, **kwargs): pass # noinspection PyUnusedLocal @post_load def make_asset_config(self, data, **kwargs): return AssetConfig(**data)
class ExpectationConfigurationBuilderConfigSchema(NotNullSchema): class Meta: unknown = INCLUDE __config_class__ = ExpectationConfigurationBuilderConfig module_name = fields.String( required=False, allow_none=True, missing= "great_expectations.rule_based_profiler.expectation_configuration_builder", ) class_name = fields.String( required=True, allow_none=False, ) expectation_type = fields.Str( required=True, error_messages={ "required": "expectation_type missing in expectation configuration builder" }, ) meta = fields.Dict( keys=fields.String( required=True, allow_none=False, ), required=False, allow_none=True, ) validation_parameter_builder_configs = fields.List( cls_or_instance=fields.Nested( lambda: ParameterBuilderConfigSchema(), required=True, allow_none=False, ), required=False, allow_none=True, )
class ExpectationValidationResultSchema(Schema): success = fields.Bool(required=False, allow_none=True) expectation_config = fields.Nested(lambda: ExpectationConfigurationSchema, required=False, allow_none=True) result = fields.Dict(required=False, allow_none=True) meta = fields.Dict(required=False, allow_none=True) exception_info = fields.Dict(required=False, allow_none=True) rendered_content = fields.List( fields.Nested(lambda: RenderedAtomicContentSchema, required=False, allow_none=True)) # noinspection PyUnusedLocal @pre_dump def convert_result_to_serializable(self, data, **kwargs): data = deepcopy(data) if isinstance(data, ExpectationValidationResult): data.result = convert_to_json_serializable(data.result) elif isinstance(data, dict): data["result"] = convert_to_json_serializable(data.get("result")) return data REMOVE_KEYS_IF_NONE = ["rendered_content"] @post_dump def clean_null_attrs(self, data: dict, **kwargs: dict) -> dict: """Removes the attributes in ExpectationValidationResultSchema.REMOVE_KEYS_IF_NONE during serialization if their values are None.""" data = deepcopy(data) for key in ExpectationConfigurationSchema.REMOVE_KEYS_IF_NONE: if key in data and data[key] is None: data.pop(key) return data # noinspection PyUnusedLocal @post_load def make_expectation_validation_result(self, data, **kwargs): return ExpectationValidationResult(**data)
class DataConnectorConfigSchema(Schema): class Meta: unknown = INCLUDE class_name = fields.String(required=True) module_name = fields.String( missing="great_expectations.datasource.data_connector") assets = fields.Dict( keys=fields.Str(), values=fields.Nested(AssetConfigSchema, required=False, allow_none=True), required=False, allow_none=True, ) base_directory = fields.String(required=False, allow_none=True) glob_directive = fields.String(required=False, allow_none=True) default_regex = fields.Dict(required=False, allow_none=True) runtime_keys = fields.List(cls_or_instance=fields.Str(), required=False, allow_none=True) bucket = fields.String(required=False, allow_none=True) prefix = fields.String(required=False, allow_none=True) delimiter = fields.String(required=False, allow_none=True) max_keys = fields.Integer(required=False, allow_none=True) boto3_options = fields.Dict(keys=fields.Str(), values=fields.Str(), required=False, allow_none=True) data_asset_name_prefix = fields.String(required=False, allow_none=True) data_asset_name_suffix = fields.String(required=False, allow_none=True) include_schema_name = fields.Boolean(required=False, allow_none=True) splitter_method = fields.String(required=False, allow_none=True) splitter_kwargs = fields.Dict(required=False, allow_none=True) sampling_method = fields.String(required=False, allow_none=True) sampling_kwargs = fields.Dict(required=False, allow_none=True) excluded_tables = fields.List(cls_or_instance=fields.Str(), required=False, allow_none=True) included_tables = fields.List(cls_or_instance=fields.Str(), required=False, allow_none=True) skip_inapplicable_tables = fields.Boolean(required=False, allow_none=True) @validates_schema def validate_schema(self, data, **kwargs): # If a class_name begins with the dollar sign ("$"), then it is assumed to be a variable name to be substituted. if data["class_name"][0] == "$": return if ("default_regex" in data) and not (data["class_name"] in [ "InferredAssetFilesystemDataConnector", "ConfiguredAssetFilesystemDataConnector", "InferredAssetS3DataConnector", "ConfiguredAssetS3DataConnector", ]): raise ge_exceptions.InvalidConfigError( f"""Your current configuration uses one or more keys in a data connector, that are required only by a subclass of the FilePathDataConnector class (your data conntector is "{data['class_name']}"). Please update your configuration to continue. """) if ("glob_directive" in data) and not (data["class_name"] in [ "InferredAssetFilesystemDataConnector", "ConfiguredAssetFilesystemDataConnector", ]): raise ge_exceptions.InvalidConfigError( f"""Your current configuration uses one or more keys in a data connector, that are required only by a filesystem type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to continue. """) if ("bucket" in data or "prefix" in data or "delimiter" in data or "max_keys" in data) and not (data["class_name"] in [ "InferredAssetS3DataConnector", "ConfiguredAssetS3DataConnector", ]): raise ge_exceptions.InvalidConfigError( f"""Your current configuration uses one or more keys in a data connector, that are required only by an S3 type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to continue. """) if ("data_asset_name_prefix" in data or "data_asset_name_suffix" in data or "include_schema_name" in data or "splitter_method" in data or "splitter_kwargs" in data or "sampling_method" in data or "sampling_kwargs" in data or "excluded_tables" in data or "included_tables" in data or "skip_inapplicable_tables" in data) and not (data["class_name"] in [ "InferredAssetSqlDataConnector", "ConfiguredAssetSqlDataConnector", ]): raise ge_exceptions.InvalidConfigError( f"""Your current configuration uses one or more keys in a data connector, that are required only by an SQL type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to continue. """) # noinspection PyUnusedLocal @post_load def make_data_connector_config(self, data, **kwargs): return DataConnectorConfig(**data)