def __init__(self, store_backend=None, runtime_environment=None, store_name=None): self._expectationSuiteValidationResultSchema = ( ExpectationSuiteValidationResultSchema()) if store_backend is not None: store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "InMemoryStoreBackend") verify_dynamic_loading_support( module_name=store_backend_module_name) store_backend_class = load_class(store_backend_class_name, store_backend_module_name) # Store Backend Class was loaded successfully; verify that it is of a correct subclass. if issubclass(store_backend_class, TupleStoreBackend): # Provide defaults for this common case store_backend["filepath_suffix"] = store_backend.get( "filepath_suffix", ".json") elif issubclass(store_backend_class, DatabaseStoreBackend): # Provide defaults for this common case store_backend["table_name"] = store_backend.get( "table_name", "ge_validations_store") store_backend["key_columns"] = store_backend.get( "key_columns", [ "expectation_suite_name", "run_name", "run_time", "batch_identifier", ], ) super().__init__( store_backend=store_backend, runtime_environment=runtime_environment, store_name=store_name, ) # Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter # out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary. self._config = { "store_backend": store_backend, "runtime_environment": runtime_environment, "store_name": store_name, "module_name": self.__class__.__module__, "class_name": self.__class__.__name__, } filter_properties_dict(properties=self._config, clean_falsy=True, inplace=True)
def __init__( self, store_name: str, store_backend: Optional[dict] = None, overwrite_existing: bool = False, runtime_environment: Optional[dict] = None, ) -> None: if not issubclass(self._configuration_class, BaseYamlConfig): raise ge_exceptions.DataContextError( "Invalid configuration: A configuration_class needs to inherit from the BaseYamlConfig class." ) if store_backend is not None: store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "InMemoryStoreBackend") verify_dynamic_loading_support( module_name=store_backend_module_name) store_backend_class = load_class(store_backend_class_name, store_backend_module_name) # Store Backend Class was loaded successfully; verify that it is of a correct subclass. if issubclass(store_backend_class, TupleStoreBackend): # Provide defaults for this common case store_backend["filepath_suffix"] = store_backend.get( "filepath_suffix", ".yml") super().__init__( store_backend=store_backend, runtime_environment=runtime_environment, store_name=store_name, ) # Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter # out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary. self._config = { "store_name": store_name, "store_backend": store_backend, "overwrite_existing": overwrite_existing, "runtime_environment": runtime_environment, "module_name": self.__class__.__module__, "class_name": self.__class__.__name__, } filter_properties_dict(properties=self._config, clean_falsy=True, inplace=True) self._overwrite_existing = overwrite_existing
def build_configuration(cls, class_name, module_name="great_expectations.datasource", data_asset_type=None, generators=None, **kwargs): """ Build a full configuration object for a datasource, potentially including generators with defaults. Args: class_name: The name of the class for which to build the config module_name: The name of the module in which the datasource class is located data_asset_type: A ClassConfig dictionary generators: Generator configuration dictionary **kwargs: Additional kwargs to be part of the datasource constructor's initialization Returns: A complete datasource configuration. """ class_ = load_class(class_name=class_name, module_name=module_name) configuration = class_.build_configuration(data_asset_type=data_asset_type, generators=generators, **kwargs) return configuration
def __init__(self, store_backend=None, runtime_environment=None): self._expectationSuiteSchema = ExpectationSuiteSchema() if store_backend is not None: store_backend_module_name = store_backend.get("module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend") store_backend_class = load_class(store_backend_class_name, store_backend_module_name) if issubclass(store_backend_class, TupleStoreBackend): # Provide defaults for this common case store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json") elif issubclass(store_backend_class, DatabaseStoreBackend): # Provide defaults for this common case store_backend["table_name"] = store_backend.get("table_name", "ge_expectations_store") store_backend["key_columns"] = store_backend.get( "key_columns", ["expectation_suite_name"] ) super(ExpectationsStore, self).__init__(store_backend=store_backend, runtime_environment=runtime_environment)
def __init__(self, store_backend=None, runtime_environment=None, store_name=None): self._expectationSuiteValidationResultSchema = ( ExpectationSuiteValidationResultSchema() ) if store_backend is not None: store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store" ) store_backend_class_name = store_backend.get( "class_name", "InMemoryStoreBackend" ) verify_dynamic_loading_support(module_name=store_backend_module_name) store_backend_class = load_class( store_backend_class_name, store_backend_module_name ) # Store Backend Class was loaded successfully; verify that it is of a correct subclass. if issubclass(store_backend_class, TupleStoreBackend): # Provide defaults for this common case store_backend["filepath_suffix"] = store_backend.get( "filepath_suffix", ".json" ) elif issubclass(store_backend_class, DatabaseStoreBackend): # Provide defaults for this common case store_backend["table_name"] = store_backend.get( "table_name", "ge_validations_store" ) store_backend["key_columns"] = store_backend.get( "key_columns", [ "expectation_suite_name", "run_name", "run_time", "batch_identifier", ], ) super().__init__( store_backend=store_backend, runtime_environment=runtime_environment, store_name=store_name, )
def __init__(self, store_backend=None, runtime_environment=None): store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "TupleFilesystemStoreBackend") verify_dynamic_loading_support(module_name=store_backend_module_name) store_class = load_class(store_backend_class_name, store_backend_module_name) # Store Class was loaded successfully; verify that it is of a correct subclass. if not issubclass(store_class, TupleStoreBackend): raise DataContextError( "Invalid configuration: HtmlSiteStore needs a TupleStoreBackend" ) if "filepath_template" in store_backend or ( "fixed_length_key" in store_backend and store_backend["fixed_length_key"] is True): logger.warning( "Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: " "filepaths will be selected based on the type of asset rendered." ) # One thing to watch for is reversibility of keys. # If several types are being written to overlapping directories, we could get collisions. module_name = 'great_expectations.data_context.store' filepath_prefix = 'expectations' filepath_suffix = '.html' expectation_suite_identifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_prefix": filepath_prefix, "filepath_suffix": filepath_suffix, }) if not expectation_suite_identifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_prefix = 'validations' validation_result_idendifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_prefix": filepath_prefix, "filepath_suffix": filepath_suffix, }) if not validation_result_idendifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_template = 'index.html' index_page_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_template": filepath_template, }) if not index_page_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_template = None static_assets_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_template": filepath_template, }) if not static_assets_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) self.store_backends = { ExpectationSuiteIdentifier: expectation_suite_identifier_obj, ValidationResultIdentifier: validation_result_idendifier_obj, "index_page": index_page_obj, "static_assets": static_assets_obj, } # NOTE: Instead of using the filesystem as the source of record for keys, # this class tracks keys separately in an internal set. # This means that keys are stored for a specific session, but can't be fetched after the original # HtmlSiteStore instance leaves scope. # Doing it this way allows us to prevent namespace collisions among keys while still having multiple # backends that write to the same directory structure. # It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it # can't necessarily set and list_keys like most other Stores. self.keys = set()
def __init__(self, root_directory, serialization_type=None, store_backend=None): self.key_class = SiteSectionIdentifier store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "FixedLengthTupleFilesystemStoreBackend") store_class = load_class(store_backend_class_name, store_backend_module_name) if not issubclass(store_class, FixedLengthTupleStoreBackend): raise DataContextError( "Invalid configuration: HtmlSiteStore needs a FixedLengthTupleStoreBackend" ) if "filepath_template" in store_backend or "key_length" in store_backend: logger.warning( "Configuring a filepath_template or key_length is not supported in SiteBuilder: " "filepaths will be selected based on the type of asset rendered." ) # # Each key type gets its own backend. # # If backends were DB connections, this could be inefficient, but it doesn't much matter for filepaths. # # One thing to watch for is reversibility of keys. # # If several types are being writtten to overlapping directories, we could get collisions. # expectations_backend_config = copy.deepcopy(store_backend) # if "base_directory" in expectations_backend_config: # expectations_backend_config["base_directory"] = os.path.join(expectations_backend_config["base_directory"], "expectations") # elif "prefix" in expectations_backend_config: # expectations_backend_config["prefix"] = os.path.join(expectations_backend_config["prefix"], "expectations") # # validations_backend_config = copy.deepcopy(store_backend) # if "base_directory" in validations_backend_config: # validations_backend_config["base_directory"] = os.path.join(validations_backend_config["base_directory"], "validations") # elif "prefix" in validations_backend_config: # validations_backend_config["prefix"] = os.path.join(validations_backend_config["prefix"], "validations") self.store_backends = { ExpectationSuiteIdentifier: instantiate_class_from_config( config=store_backend, runtime_config={"root_directory": root_directory}, config_defaults={ "module_name": "great_expectations.data_context.store", "key_length": 4, "filepath_template": 'expectations/{0}/{1}/{2}/{3}.html', }), ValidationResultIdentifier: instantiate_class_from_config( config=store_backend, runtime_config={"root_directory": root_directory}, config_defaults={ "module_name": "great_expectations.data_context.store", "key_length": 5, "filepath_template": 'validations/{4}/{0}/{1}/{2}/{3}.html', }), "index_page": instantiate_class_from_config( config=store_backend, runtime_config={"root_directory": root_directory}, config_defaults={ "module_name": "great_expectations.data_context.store", "key_length": 0, "filepath_template": 'index.html', }), } self.root_directory = root_directory self.serialization_type = serialization_type # NOTE: Instead of using the filesystem as the source of record for keys, # this class trackes keys separately in an internal set. # This means that keys are stored for a specific session, but can't be fetched after the original # HtmlSiteStore instance leaves scope. # Doing it this way allows us to prevent namespace collisions among keys while still having multiple # backends that write to the same directory structure. # It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it # can't necessarily set and list_keys like most other Stores. self.keys = set()
def __init__(self, store_backend=None, runtime_environment=None): store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "TupleFilesystemStoreBackend") verify_dynamic_loading_support(module_name=store_backend_module_name) store_class = load_class(store_backend_class_name, store_backend_module_name) # Store Class was loaded successfully; verify that it is of a correct subclass. if not issubclass(store_class, (TupleStoreBackend, GeCloudStoreBackend)): raise DataContextError( "Invalid configuration: HtmlSiteStore needs a TupleStoreBackend or GeCloudStoreBackend" ) if "filepath_template" in store_backend or ( "fixed_length_key" in store_backend and store_backend["fixed_length_key"] is True): logger.warning( "Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: " "filepaths will be selected based on the type of asset rendered." ) # One thing to watch for is reversibility of keys. # If several types are being written to overlapping directories, we could get collisions. module_name = "great_expectations.data_context.store" filepath_suffix = ".html" is_ge_cloud_store = store_backend[ "class_name"] == "GeCloudStoreBackend" expectation_config_defaults = { "module_name": module_name, "filepath_prefix": "expectations", "filepath_suffix": filepath_suffix, "suppress_store_backend_id": True, } if is_ge_cloud_store: expectation_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } expectation_suite_identifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=expectation_config_defaults, ) if not expectation_suite_identifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) validation_result_config_defaults = { "module_name": module_name, "filepath_prefix": "validations", "filepath_suffix": filepath_suffix, "suppress_store_backend_id": True, } if is_ge_cloud_store: validation_result_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } validation_result_idendifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=validation_result_config_defaults, ) if not validation_result_idendifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) filepath_template = "index.html" index_page_config_defaults = { "module_name": module_name, "filepath_template": filepath_template, "suppress_store_backend_id": True, } if is_ge_cloud_store: index_page_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } index_page_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=index_page_config_defaults, ) if not index_page_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) static_assets_config_defaults = { "module_name": module_name, "filepath_template": None, "suppress_store_backend_id": True, } if is_ge_cloud_store: static_assets_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } static_assets_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=static_assets_config_defaults, ) if not static_assets_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) self.store_backends = { ExpectationSuiteIdentifier: expectation_suite_identifier_obj, ValidationResultIdentifier: validation_result_idendifier_obj, "index_page": index_page_obj, "static_assets": static_assets_obj, } # NOTE: Instead of using the filesystem as the source of record for keys, # this class tracks keys separately in an internal set. # This means that keys are stored for a specific session, but can't be fetched after the original # HtmlSiteStore instance leaves scope. # Doing it this way allows us to prevent namespace collisions among keys while still having multiple # backends that write to the same directory structure. # It's a pretty reasonable way for HtmlSiteStore to do its job---you just have to remember that it # can't necessarily set and list_keys like most other Stores. self.keys = set() # Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter # out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary. self._config = { "store_backend": store_backend, "runtime_environment": runtime_environment, "module_name": self.__class__.__module__, "class_name": self.__class__.__name__, } filter_properties_dict(properties=self._config, clean_falsy=True, inplace=True)
def test_load_class_raises_error_when_module_not_found(): with pytest.raises(gee.PluginModuleNotFoundError): load_class("foo", "bar")
def test_load_class_raises_error_when_class_not_found(): with pytest.raises(gee.PluginClassNotFoundError): load_class("TotallyNotARealClass", "great_expectations.datasource")