def __init__(
self,
name,
data_context,
target_store,
source_store_name,
custom_styles_directory=None,
show_how_to_buttons=True,
run_id_filter=None,
validation_results_limit=None,
renderer=None,
view=None,
data_context_id=None,
**kwargs,
):
self.name = name
self.source_store = data_context.stores[source_store_name]
self.target_store = target_store
self.run_id_filter = run_id_filter
self.validation_results_limit = validation_results_limit
self.data_context_id = data_context_id
self.show_how_to_buttons = show_how_to_buttons
if renderer is None:
raise exceptions.InvalidConfigError(
"SiteSectionBuilder requires a renderer configuration with a class_name key."
)
module_name = (
renderer.get("module_name") or "great_expectations.render.renderer"
)
self.renderer_class = instantiate_class_from_config(
config=renderer,
runtime_environment={"data_context": data_context},
config_defaults={"module_name": module_name},
)
if not self.renderer_class:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer["class_name"],
)
module_name = "great_expectations.render.view"
if view is None:
view = {
"module_name": module_name,
"class_name": "DefaultJinjaPageView",
}
module_name = view.get("module_name") or module_name
self.view_class = instantiate_class_from_config(
config=view,
runtime_environment={"custom_styles_directory": custom_styles_directory},
config_defaults={"module_name": module_name},
)
if not self.view_class:
raise exceptions.ClassInstantiationError(
module_name=view["module_name"],
package_name=None,
class_name=view["class_name"],
)
def __init__(self,
name,
site_name,
data_context,
target_store,
custom_styles_directory=None,
show_how_to_buttons=True,
validation_results_limit=None,
renderer=None,
view=None,
data_context_id=None,
**kwargs):
# NOTE: This method is almost identical to DefaultSiteSectionBuilder
self.name = name
self.site_name = site_name
self.data_context = data_context
self.target_store = target_store
self.validation_results_limit = validation_results_limit
self.data_context_id = data_context_id
self.show_how_to_buttons = show_how_to_buttons
if renderer is None:
renderer = {
"module_name": "great_expectations.render.renderer",
"class_name": "SiteIndexPageRenderer",
}
module_name = renderer.get(
'module_name') or 'great_expectations.render.renderer'
self.renderer_class = instantiate_class_from_config(
config=renderer,
runtime_environment={"data_context": data_context},
config_defaults={"module_name": module_name})
if not self.renderer_class:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer['class_name'])
module_name = 'great_expectations.render.view'
if view is None:
view = {
"module_name": module_name,
"class_name": "DefaultJinjaIndexPageView",
}
module_name = view.get('module_name') or module_name
self.view_class = instantiate_class_from_config(
config=view,
runtime_environment={
"custom_styles_directory": custom_styles_directory
},
config_defaults={"module_name": module_name})
if not self.view_class:
raise exceptions.ClassInstantiationError(
module_name=view['module_name'],
package_name=None,
class_name=view['class_name'])
def _build_datasource_from_config(
self,
name: str,
config: dict,
) -> BaseDatasource:
module_name: str = "great_expectations.datasource"
runtime_environment: dict = {
"name": name,
"data_context_root_directory": self.root_directory,
}
new_datasource: BaseDatasource = instantiate_class_from_config(
config=config,
runtime_environment=runtime_environment,
config_defaults={"module_name": module_name},
)
if not new_datasource:
raise ge_exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=config["class_name"],
)
if not isinstance(new_datasource, BaseDatasource):
raise TypeError(
f"Newly instantiated component {name} is not an instance of BaseDatasource. Please check class_name in the config."
)
self._cached_datasources[name] = new_datasource
return new_datasource
def build_store_from_config(
store_name: str = None,
store_config: dict = None,
module_name: str = "great_expectations.data_context.store",
runtime_environment: dict = None,
):
if store_config is None or module_name is None:
return None
try:
config_defaults: dict = {
"store_name": store_name,
"module_name": module_name,
}
new_store = instantiate_class_from_config(
config=store_config,
runtime_environment=runtime_environment,
config_defaults=config_defaults,
)
except ge_exceptions.DataContextError as e:
new_store = None
logger.critical(
f"Error {e} occurred while attempting to instantiate a store.")
if not new_store:
class_name: str = store_config["class_name"]
module_name = store_config["module_name"]
raise ge_exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=class_name,
)
return new_store
def _build_asset_from_config(self, config: dict):
runtime_environment: dict = {"data_connector": self}
config = assetConfigSchema.load(config)
config = assetConfigSchema.dump(config)
asset: Asset = instantiate_class_from_config(
config=config,
runtime_environment=runtime_environment,
config_defaults={},
)
if not asset:
raise ge_exceptions.ClassInstantiationError(
module_name="great_expectations.datasource.data_connector.asset",
package_name=None,
class_name=config["class_name"],
)
return asset
def _build_asset_from_config(self, name: str, config: dict):
"""Build an Asset using the provided configuration and return the newly-built Asset."""
runtime_environment: dict = {"name": name, "data_connector": self}
asset: Asset = instantiate_class_from_config(
config=config,
runtime_environment=runtime_environment,
config_defaults={
"module_name": "great_expectations.datasource.data_connector.asset",
"class_name": "Asset",
},
)
if not asset:
raise ge_exceptions.ClassInstantiationError(
module_name="great_expectations.datasource.data_connector.asset",
package_name=None,
class_name=config["class_name"],
)
return asset
def _build_asset_from_config(runtime_environment: "DataConnector",
config: dict) -> Asset:
"""Build Asset from configuration and return asset. Used by both ConfiguredAssetDataConnector and RuntimeDataConnector"""
runtime_environment: dict = {"data_connector": runtime_environment}
config = assetConfigSchema.load(config)
config = assetConfigSchema.dump(config)
asset: Asset = instantiate_class_from_config(
config=config,
runtime_environment=runtime_environment,
config_defaults={},
)
if not asset:
raise ge_exceptions.ClassInstantiationError(
module_name="great_expectations.datasource.data_connector.asset",
package_name=None,
class_name=config["class_name"],
)
return asset
def __init__(
self,
data_context,
action_list,
name,
result_format={"result_format": "SUMMARY"},
):
super().__init__()
self.data_context = data_context
self.name = name
result_format = parse_result_format(result_format)
assert result_format["result_format"] in [
"BOOLEAN_ONLY",
"BASIC",
"SUMMARY",
"COMPLETE",
]
self.result_format = result_format
self.action_list = action_list
self.actions = OrderedDict()
for action_config in action_list:
assert isinstance(action_config, dict)
# NOTE: Eugene: 2019-09-23: need a better way to validate an action config:
if not set(action_config.keys()) == {"name", "action"}:
raise KeyError(
'Action config keys must be ("name", "action"). Instead got {}'
.format(action_config.keys()))
config = action_config["action"]
module_name = "great_expectations.validation_operators"
new_action = instantiate_class_from_config(
config=config,
runtime_environment={"data_context": self.data_context},
config_defaults={"module_name": module_name},
)
if not new_action:
raise ge_exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=config["class_name"],
)
self.actions[action_config["name"]] = new_action
def __init__(
self,
data_context,
store_backend,
site_name=None,
site_index_builder=None,
show_how_to_buttons=True,
site_section_builders=None,
runtime_environment=None,
**kwargs,
):
self.site_name = site_name
self.data_context = data_context
self.store_backend = store_backend
self.show_how_to_buttons = show_how_to_buttons
usage_statistics_config = data_context.anonymous_usage_statistics
data_context_id = None
if (
usage_statistics_config
and usage_statistics_config.enabled
and usage_statistics_config.data_context_id
):
data_context_id = usage_statistics_config.data_context_id
self.data_context_id = data_context_id
# set custom_styles_directory if present
custom_styles_directory = None
plugins_directory = data_context.plugins_directory
if plugins_directory and os.path.isdir(
os.path.join(plugins_directory, "custom_data_docs", "styles")
):
custom_styles_directory = os.path.join(
plugins_directory, "custom_data_docs", "styles"
)
# set custom_views_directory if present
custom_views_directory = None
if plugins_directory and os.path.isdir(
os.path.join(plugins_directory, "custom_data_docs", "views")
):
custom_views_directory = os.path.join(
plugins_directory, "custom_data_docs", "views"
)
if site_index_builder is None:
site_index_builder = {"class_name": "DefaultSiteIndexBuilder"}
# The site builder is essentially a frontend store. We'll open up
# three types of backends using the base
# type of the configuration defined in the store_backend section
self.target_store = HtmlSiteStore(
store_backend=store_backend, runtime_environment=runtime_environment
)
default_site_section_builders_config = {
"expectations": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.expectations_store_name,
"renderer": {"class_name": "ExpectationSuitePageRenderer"},
},
"validations": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.validations_store_name,
"run_name_filter": {"ne": "profiling"},
"renderer": {"class_name": "ValidationResultsPageRenderer"},
"validation_results_limit": site_index_builder.get(
"validation_results_limit"
),
},
"profiling": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.validations_store_name,
"run_name_filter": {"eq": "profiling"},
"renderer": {"class_name": "ProfilingResultsPageRenderer"},
},
}
if site_section_builders is None:
site_section_builders = default_site_section_builders_config
else:
site_section_builders = nested_update(
default_site_section_builders_config, site_section_builders
)
self.site_section_builders = {}
for site_section_name, site_section_config in site_section_builders.items():
if not site_section_config or site_section_config in [
"0",
"None",
"False",
"false",
"FALSE",
"none",
"NONE",
]:
continue
module_name = (
site_section_config.get("module_name")
or "great_expectations.render.renderer.site_builder"
)
self.site_section_builders[
site_section_name
] = instantiate_class_from_config(
config=site_section_config,
runtime_environment={
"data_context": data_context,
"target_store": self.target_store,
"custom_styles_directory": custom_styles_directory,
"custom_views_directory": custom_views_directory,
"data_context_id": self.data_context_id,
"show_how_to_buttons": self.show_how_to_buttons,
},
config_defaults={"name": site_section_name, "module_name": module_name},
)
if not self.site_section_builders[site_section_name]:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=site_section_config["class_name"],
)
module_name = (
site_index_builder.get("module_name")
or "great_expectations.render.renderer.site_builder"
)
class_name = site_index_builder.get("class_name") or "DefaultSiteIndexBuilder"
self.site_index_builder = instantiate_class_from_config(
config=site_index_builder,
runtime_environment={
"data_context": data_context,
"custom_styles_directory": custom_styles_directory,
"custom_views_directory": custom_views_directory,
"show_how_to_buttons": self.show_how_to_buttons,
"target_store": self.target_store,
"site_name": self.site_name,
"data_context_id": self.data_context_id,
"source_stores": {
section_name: section_config.get("source_store_name")
for (section_name, section_config) in site_section_builders.items()
},
},
config_defaults={
"name": "site_index_builder",
"module_name": module_name,
"class_name": class_name,
},
)
if not self.site_index_builder:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=site_index_builder["class_name"],
)
def __init__(self,
data_context,
store_backend,
site_name=None,
site_index_builder=None,
show_how_to_buttons=True,
site_section_builders=None,
runtime_environment=None,
**kwargs
):
self.site_name = site_name
self.data_context = data_context
self.store_backend = store_backend
self.show_how_to_buttons = show_how_to_buttons
# set custom_styles_directory if present
custom_styles_directory = None
plugins_directory = data_context.plugins_directory
if plugins_directory and os.path.isdir(os.path.join(plugins_directory, "custom_data_docs", "styles")):
custom_styles_directory = os.path.join(plugins_directory, "custom_data_docs", "styles")
# The site builder is essentially a frontend store. We'll open up three types of backends using the base
# type of the configuration defined in the store_backend section
self.target_store = HtmlSiteStore(
store_backend=store_backend,
runtime_environment=runtime_environment
)
if site_index_builder is None:
site_index_builder = {
"class_name": "DefaultSiteIndexBuilder"
}
module_name = site_index_builder.get('module_name') or 'great_expectations.render.renderer.site_builder'
class_name = site_index_builder.get('class_name') or 'DefaultSiteIndexBuilder'
self.site_index_builder = instantiate_class_from_config(
config=site_index_builder,
runtime_environment={
"data_context": data_context,
"custom_styles_directory": custom_styles_directory,
"show_how_to_buttons": self.show_how_to_buttons,
"target_store": self.target_store,
"site_name": self.site_name
},
config_defaults={
"name": "site_index_builder",
"module_name": module_name,
"class_name": class_name
}
)
if not self.site_index_builder:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=site_index_builder['class_name']
)
default_site_section_builders_config = {
"expectations": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.expectations_store_name,
"renderer": {
"class_name": "ExpectationSuitePageRenderer"
}
},
"validations": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.validations_store_name,
"run_id_filter": {
"ne": "profiling"
},
"renderer": {
"class_name": "ValidationResultsPageRenderer"
},
"validation_results_limit": site_index_builder.get("validation_results_limit")
},
"profiling": {
"class_name": "DefaultSiteSectionBuilder",
"source_store_name": data_context.validations_store_name,
"run_id_filter": {
"eq": "profiling"
},
"renderer": {
"class_name": "ProfilingResultsPageRenderer"
}
}
}
if site_section_builders is None:
site_section_builders = default_site_section_builders_config
else:
site_section_builders = nested_update(default_site_section_builders_config, site_section_builders)
self.site_section_builders = {}
for site_section_name, site_section_config in site_section_builders.items():
if not site_section_config or site_section_config in \
['0', 'None', 'False', 'false', 'FALSE', 'none', 'NONE']:
continue
module_name = site_section_config.get('module_name') or 'great_expectations.render.renderer.site_builder'
self.site_section_builders[site_section_name] = instantiate_class_from_config(
config=site_section_config,
runtime_environment={
"data_context": data_context,
"target_store": self.target_store,
"custom_styles_directory": custom_styles_directory,
"show_how_to_buttons": self.show_how_to_buttons,
},
config_defaults={
"name": site_section_name,
"module_name": module_name
}
)
if not self.site_section_builders[site_section_name]:
raise exceptions.ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=site_section_config['class_name']
)
