def __init__(self,
store_backend=None,
runtime_environment=None,
store_name=None):
self._expectationSuiteValidationResultSchema = (
ExpectationSuiteValidationResultSchema())
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(
module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get(
"filepath_suffix", ".json")
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get(
"table_name", "ge_validations_store")
store_backend["key_columns"] = store_backend.get(
"key_columns",
[
"expectation_suite_name",
"run_name",
"run_time",
"batch_identifier",
],
)
super().__init__(
store_backend=store_backend,
runtime_environment=runtime_environment,
store_name=store_name,
)
# Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter
# out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary.
self._config = {
"store_backend": store_backend,
"runtime_environment": runtime_environment,
"store_name": store_name,
"module_name": self.__class__.__module__,
"class_name": self.__class__.__name__,
}
filter_properties_dict(properties=self._config,
clean_falsy=True,
inplace=True)
def __init__(
self,
store_name: str,
store_backend: Optional[dict] = None,
overwrite_existing: bool = False,
runtime_environment: Optional[dict] = None,
) -> None:
if not issubclass(self._configuration_class, BaseYamlConfig):
raise ge_exceptions.DataContextError(
"Invalid configuration: A configuration_class needs to inherit from the BaseYamlConfig class."
)
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(
module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get(
"filepath_suffix", ".yml")
super().__init__(
store_backend=store_backend,
runtime_environment=runtime_environment,
store_name=store_name,
)
# Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter
# out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary.
self._config = {
"store_name": store_name,
"store_backend": store_backend,
"overwrite_existing": overwrite_existing,
"runtime_environment": runtime_environment,
"module_name": self.__class__.__module__,
"class_name": self.__class__.__name__,
}
filter_properties_dict(properties=self._config,
clean_falsy=True,
inplace=True)
self._overwrite_existing = overwrite_existing
def build_configuration(cls, class_name, module_name="great_expectations.datasource", data_asset_type=None, generators=None, **kwargs):
"""
Build a full configuration object for a datasource, potentially including generators with defaults.
Args:
class_name: The name of the class for which to build the config
module_name: The name of the module in which the datasource class is located
data_asset_type: A ClassConfig dictionary
generators: Generator configuration dictionary
**kwargs: Additional kwargs to be part of the datasource constructor's initialization
Returns:
A complete datasource configuration.
"""
class_ = load_class(class_name=class_name, module_name=module_name)
configuration = class_.build_configuration(data_asset_type=data_asset_type, generators=generators, **kwargs)
return configuration
def __init__(self, store_backend=None, runtime_environment=None):
self._expectationSuiteSchema = ExpectationSuiteSchema()
if store_backend is not None:
store_backend_module_name = store_backend.get("module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend")
store_backend_class = load_class(store_backend_class_name, store_backend_module_name)
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get("table_name", "ge_expectations_store")
store_backend["key_columns"] = store_backend.get(
"key_columns", ["expectation_suite_name"]
)
super(ExpectationsStore, self).__init__(store_backend=store_backend, runtime_environment=runtime_environment)
def __init__(self, store_backend=None, runtime_environment=None, store_name=None):
self._expectationSuiteValidationResultSchema = (
ExpectationSuiteValidationResultSchema()
)
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store"
)
store_backend_class_name = store_backend.get(
"class_name", "InMemoryStoreBackend"
)
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_backend_class = load_class(
store_backend_class_name, store_backend_module_name
)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get(
"filepath_suffix", ".json"
)
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get(
"table_name", "ge_validations_store"
)
store_backend["key_columns"] = store_backend.get(
"key_columns",
[
"expectation_suite_name",
"run_name",
"run_time",
"batch_identifier",
],
)
super().__init__(
store_backend=store_backend,
runtime_environment=runtime_environment,
store_name=store_name,
)
def __init__(self, store_backend=None, runtime_environment=None):
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "TupleFilesystemStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Class was loaded successfully; verify that it is of a correct subclass.
if not issubclass(store_class, TupleStoreBackend):
raise DataContextError(
"Invalid configuration: HtmlSiteStore needs a TupleStoreBackend"
)
if "filepath_template" in store_backend or (
"fixed_length_key" in store_backend
and store_backend["fixed_length_key"] is True):
logger.warning(
"Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: "
"filepaths will be selected based on the type of asset rendered."
)
# One thing to watch for is reversibility of keys.
# If several types are being written to overlapping directories, we could get collisions.
module_name = 'great_expectations.data_context.store'
filepath_prefix = 'expectations'
filepath_suffix = '.html'
expectation_suite_identifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_prefix": filepath_prefix,
"filepath_suffix": filepath_suffix,
})
if not expectation_suite_identifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_prefix = 'validations'
validation_result_idendifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_prefix": filepath_prefix,
"filepath_suffix": filepath_suffix,
})
if not validation_result_idendifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_template = 'index.html'
index_page_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_template": filepath_template,
})
if not index_page_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_template = None
static_assets_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_template": filepath_template,
})
if not static_assets_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
self.store_backends = {
ExpectationSuiteIdentifier: expectation_suite_identifier_obj,
ValidationResultIdentifier: validation_result_idendifier_obj,
"index_page": index_page_obj,
"static_assets": static_assets_obj,
}
# NOTE: Instead of using the filesystem as the source of record for keys,
# this class tracks keys separately in an internal set.
# This means that keys are stored for a specific session, but can't be fetched after the original
# HtmlSiteStore instance leaves scope.
# Doing it this way allows us to prevent namespace collisions among keys while still having multiple
# backends that write to the same directory structure.
# It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it
# can't necessarily set and list_keys like most other Stores.
self.keys = set()
def __init__(self,
root_directory,
serialization_type=None,
store_backend=None):
self.key_class = SiteSectionIdentifier
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "FixedLengthTupleFilesystemStoreBackend")
store_class = load_class(store_backend_class_name,
store_backend_module_name)
if not issubclass(store_class, FixedLengthTupleStoreBackend):
raise DataContextError(
"Invalid configuration: HtmlSiteStore needs a FixedLengthTupleStoreBackend"
)
if "filepath_template" in store_backend or "key_length" in store_backend:
logger.warning(
"Configuring a filepath_template or key_length is not supported in SiteBuilder: "
"filepaths will be selected based on the type of asset rendered."
)
# # Each key type gets its own backend.
# # If backends were DB connections, this could be inefficient, but it doesn't much matter for filepaths.
# # One thing to watch for is reversibility of keys.
# # If several types are being writtten to overlapping directories, we could get collisions.
# expectations_backend_config = copy.deepcopy(store_backend)
# if "base_directory" in expectations_backend_config:
# expectations_backend_config["base_directory"] = os.path.join(expectations_backend_config["base_directory"], "expectations")
# elif "prefix" in expectations_backend_config:
# expectations_backend_config["prefix"] = os.path.join(expectations_backend_config["prefix"], "expectations")
#
# validations_backend_config = copy.deepcopy(store_backend)
# if "base_directory" in validations_backend_config:
# validations_backend_config["base_directory"] = os.path.join(validations_backend_config["base_directory"], "validations")
# elif "prefix" in validations_backend_config:
# validations_backend_config["prefix"] = os.path.join(validations_backend_config["prefix"], "validations")
self.store_backends = {
ExpectationSuiteIdentifier:
instantiate_class_from_config(
config=store_backend,
runtime_config={"root_directory": root_directory},
config_defaults={
"module_name": "great_expectations.data_context.store",
"key_length": 4,
"filepath_template": 'expectations/{0}/{1}/{2}/{3}.html',
}),
ValidationResultIdentifier:
instantiate_class_from_config(
config=store_backend,
runtime_config={"root_directory": root_directory},
config_defaults={
"module_name": "great_expectations.data_context.store",
"key_length": 5,
"filepath_template":
'validations/{4}/{0}/{1}/{2}/{3}.html',
}),
"index_page":
instantiate_class_from_config(
config=store_backend,
runtime_config={"root_directory": root_directory},
config_defaults={
"module_name": "great_expectations.data_context.store",
"key_length": 0,
"filepath_template": 'index.html',
}),
}
self.root_directory = root_directory
self.serialization_type = serialization_type
# NOTE: Instead of using the filesystem as the source of record for keys,
# this class trackes keys separately in an internal set.
# This means that keys are stored for a specific session, but can't be fetched after the original
# HtmlSiteStore instance leaves scope.
# Doing it this way allows us to prevent namespace collisions among keys while still having multiple
# backends that write to the same directory structure.
# It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it
# can't necessarily set and list_keys like most other Stores.
self.keys = set()
def __init__(self, store_backend=None, runtime_environment=None):
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "TupleFilesystemStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Class was loaded successfully; verify that it is of a correct subclass.
if not issubclass(store_class,
(TupleStoreBackend, GeCloudStoreBackend)):
raise DataContextError(
"Invalid configuration: HtmlSiteStore needs a TupleStoreBackend or GeCloudStoreBackend"
)
if "filepath_template" in store_backend or (
"fixed_length_key" in store_backend
and store_backend["fixed_length_key"] is True):
logger.warning(
"Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: "
"filepaths will be selected based on the type of asset rendered."
)
# One thing to watch for is reversibility of keys.
# If several types are being written to overlapping directories, we could get collisions.
module_name = "great_expectations.data_context.store"
filepath_suffix = ".html"
is_ge_cloud_store = store_backend[
"class_name"] == "GeCloudStoreBackend"
expectation_config_defaults = {
"module_name": module_name,
"filepath_prefix": "expectations",
"filepath_suffix": filepath_suffix,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
expectation_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
expectation_suite_identifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=expectation_config_defaults,
)
if not expectation_suite_identifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
validation_result_config_defaults = {
"module_name": module_name,
"filepath_prefix": "validations",
"filepath_suffix": filepath_suffix,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
validation_result_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
validation_result_idendifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=validation_result_config_defaults,
)
if not validation_result_idendifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
filepath_template = "index.html"
index_page_config_defaults = {
"module_name": module_name,
"filepath_template": filepath_template,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
index_page_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
index_page_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=index_page_config_defaults,
)
if not index_page_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
static_assets_config_defaults = {
"module_name": module_name,
"filepath_template": None,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
static_assets_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
static_assets_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=static_assets_config_defaults,
)
if not static_assets_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
self.store_backends = {
ExpectationSuiteIdentifier: expectation_suite_identifier_obj,
ValidationResultIdentifier: validation_result_idendifier_obj,
"index_page": index_page_obj,
"static_assets": static_assets_obj,
}
# NOTE: Instead of using the filesystem as the source of record for keys,
# this class tracks keys separately in an internal set.
# This means that keys are stored for a specific session, but can't be fetched after the original
# HtmlSiteStore instance leaves scope.
# Doing it this way allows us to prevent namespace collisions among keys while still having multiple
# backends that write to the same directory structure.
# It's a pretty reasonable way for HtmlSiteStore to do its job---you just have to remember that it
# can't necessarily set and list_keys like most other Stores.
self.keys = set()
# Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter
# out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary.
self._config = {
"store_backend": store_backend,
"runtime_environment": runtime_environment,
"module_name": self.__class__.__module__,
"class_name": self.__class__.__name__,
}
filter_properties_dict(properties=self._config,
clean_falsy=True,
inplace=True)
def test_load_class_raises_error_when_module_not_found():
with pytest.raises(gee.PluginModuleNotFoundError):
load_class("foo", "bar")
def test_load_class_raises_error_when_class_not_found():
with pytest.raises(gee.PluginClassNotFoundError):
load_class("TotallyNotARealClass", "great_expectations.datasource")
