class SampleConfigSchema(Schema):
class Meta:
unknown = INCLUDE
some_param_0 = fields.String()
some_param_1 = fields.Integer()
@validates_schema
def validate_schema(self, data, **kwargs):
pass
class DataConnectorConfigSchema(Schema):
class Meta:
unknown = INCLUDE
class_name = fields.String(required=True)
module_name = fields.String(
missing="great_expectations.datasource.data_connector")
assets = fields.Dict(
keys=fields.Str(),
values=fields.Nested(AssetConfigSchema,
required=False,
allow_none=True),
required=False,
allow_none=True,
)
base_directory = fields.String(required=False, allow_none=True)
glob_directive = fields.String(required=False, allow_none=True)
default_regex = fields.Dict(required=False, allow_none=True)
runtime_keys = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
bucket = fields.String(required=False, allow_none=True)
prefix = fields.String(required=False, allow_none=True)
delimiter = fields.String(required=False, allow_none=True)
max_keys = fields.Integer(required=False, allow_none=True)
boto3_options = fields.Dict(keys=fields.Str(),
values=fields.Str(),
required=False,
allow_none=True)
data_asset_name_prefix = fields.String(required=False, allow_none=True)
data_asset_name_suffix = fields.String(required=False, allow_none=True)
include_schema_name = fields.Boolean(required=False, allow_none=True)
splitter_method = fields.String(required=False, allow_none=True)
splitter_kwargs = fields.Dict(required=False, allow_none=True)
sampling_method = fields.String(required=False, allow_none=True)
sampling_kwargs = fields.Dict(required=False, allow_none=True)
excluded_tables = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
included_tables = fields.List(cls_or_instance=fields.Str(),
required=False,
allow_none=True)
skip_inapplicable_tables = fields.Boolean(required=False, allow_none=True)
@validates_schema
def validate_schema(self, data, **kwargs):
# If a class_name begins with the dollar sign ("$"), then it is assumed to be a variable name to be substituted.
if data["class_name"][0] == "$":
return
if ("default_regex" in data) and not (data["class_name"] in [
"InferredAssetFilesystemDataConnector",
"ConfiguredAssetFilesystemDataConnector",
"InferredAssetS3DataConnector",
"ConfiguredAssetS3DataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by a
subclass of the FilePathDataConnector class (your data conntector is "{data['class_name']}"). Please update your
configuration to continue.
""")
if ("glob_directive" in data) and not (data["class_name"] in [
"InferredAssetFilesystemDataConnector",
"ConfiguredAssetFilesystemDataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by a
filesystem type of the data connector (your data conntector is "{data['class_name']}"). Please update your
configuration to continue.
""")
if ("bucket" in data or "prefix" in data or "delimiter" in data
or "max_keys" in data) and not (data["class_name"] in [
"InferredAssetS3DataConnector",
"ConfiguredAssetS3DataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by an
S3 type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to
continue.
""")
if ("data_asset_name_prefix" in data
or "data_asset_name_suffix" in data
or "include_schema_name" in data or "splitter_method" in data
or "splitter_kwargs" in data or "sampling_method" in data
or "sampling_kwargs" in data or "excluded_tables" in data
or "included_tables" in data or "skip_inapplicable_tables"
in data) and not (data["class_name"] in [
"InferredAssetSqlDataConnector",
"ConfiguredAssetSqlDataConnector",
]):
raise ge_exceptions.InvalidConfigError(
f"""Your current configuration uses one or more keys in a data connector, that are required only by an
SQL type of the data connector (your data conntector is "{data['class_name']}"). Please update your configuration to
continue.
""")
# noinspection PyUnusedLocal
@post_load
def make_data_connector_config(self, data, **kwargs):
return DataConnectorConfig(**data)