def __init__(
self,
overview_table_renderer=None,
expectation_string_renderer=None,
runtime_environment=None,
):
super().__init__()
if overview_table_renderer is None:
overview_table_renderer = {
"class_name": "ProfilingOverviewTableContentBlockRenderer"
}
if expectation_string_renderer is None:
expectation_string_renderer = {
"class_name": "ExpectationStringRenderer"
}
module_name = "great_expectations.render.renderer.content_block"
self._overview_table_renderer = instantiate_class_from_config(
config=overview_table_renderer,
runtime_environment=runtime_environment,
config_defaults={"module_name": module_name},
)
if not self._overview_table_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=overview_table_renderer["class_name"],
)
self._expectation_string_renderer = instantiate_class_from_config(
config=expectation_string_renderer,
runtime_environment=runtime_environment,
config_defaults={"module_name": module_name},
)
if not self._expectation_string_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=expectation_string_renderer["class_name"],
)
self.content_block_function_names = [
"_render_header",
"_render_overview_table",
"_render_quantile_table",
"_render_stats_table",
"_render_values_set",
"_render_histogram",
"_render_bar_chart_table",
"_render_failed",
]
def __init__(self, properties_table_renderer=None, runtime_environment=None):
super().__init__()
if properties_table_renderer is None:
properties_table_renderer = {
"class_name": "ProfilingColumnPropertiesTableContentBlockRenderer"
}
module_name = "great_expectations.render.renderer.content_block"
self._properties_table_renderer = instantiate_class_from_config(
config=properties_table_renderer,
runtime_environment=runtime_environment,
config_defaults={"module_name": module_name},
)
if not self._properties_table_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=properties_table_renderer["class_name"],
)
self.content_block_function_names = [
"_render_header",
"_render_properties_table",
"_render_quantile_table",
"_render_stats_table",
"_render_values_set",
"_render_histogram",
"_render_value_counts_bar_chart",
"_render_failed",
]
def render(self) -> None:
"""Renders content using the:
- atomic prescriptive renderer for the expectation configuration associated with this
ExpectationValidationResult to self.expectation_config.rendered_content
- atomic diagnostic renderer for the expectation configuration associated with this
ExpectationValidationResult to self.rendered_content.
"""
inline_renderer_config: Dict[str,
Union[str,
ExpectationValidationResult]] = {
"class_name": "InlineRenderer",
"render_object": self,
}
module_name: str = "great_expectations.render.renderer.inline_renderer"
inline_renderer = instantiate_class_from_config(
config=inline_renderer_config,
runtime_environment={},
config_defaults={"module_name": module_name},
)
if not inline_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=inline_renderer_config["class_name"],
)
(
self.expectation_config.rendered_content,
self.rendered_content,
) = inline_renderer.render()
def __init__(self,
store_backend=None,
runtime_environment=None,
store_name="no_store_name"):
"""
Runtime environment may be necessary to instantiate store backend elements.
Args:
store_backend:
runtime_environment:
store_name: store name given in the DataContextConfig (via either in-code or yaml configuration)
"""
if store_backend is None:
store_backend = {"class_name": "InMemoryStoreBackend"}
self._store_name = store_name
logger.debug("Building store_backend.")
module_name = "great_expectations.data_context.store"
self._store_backend = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment or {},
config_defaults={
"module_name": module_name,
"store_name": self._store_name,
},
)
if not self._store_backend:
raise ClassInstantiationError(module_name=module_name,
package_name=None,
class_name=store_backend)
if not isinstance(self._store_backend, StoreBackend):
raise DataContextError(
"Invalid StoreBackend configuration: expected a StoreBackend instance."
)
self._use_fixed_length_key = self._store_backend.fixed_length_key
def __init__(
self,
column_section_renderer=None,
run_info_at_end: bool = False,
data_context=None,
) -> None:
"""
Args:
column_section_renderer:
run_info_at_end: Move the run info (Info, Batch Markers, Batch Kwargs) to the end
of the rendered output rather than after Statistics.
"""
super().__init__()
if column_section_renderer is None:
column_section_renderer = {
"class_name": "ValidationResultsColumnSectionRenderer"
}
module_name = "great_expectations.render.renderer.column_section_renderer"
self._column_section_renderer = instantiate_class_from_config(
config=column_section_renderer,
runtime_environment={},
config_defaults={
"module_name":
column_section_renderer.get("module_name", module_name)
},
)
if not self._column_section_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=column_section_renderer["class_name"],
)
self.run_info_at_end = run_info_at_end
self._data_context = data_context
def __init__(
self,
name="default",
datasource=None,
bucket=None,
boto3_options=None,
base_directory="/data",
reader_options=None,
known_extensions=None,
reader_method=None,
):
super().__init__(name, datasource=datasource)
if not s3fs:
raise ClassInstantiationError(
"ModuleNotFoundError: No module named 's3fs'")
if reader_options is None:
reader_options = self._default_reader_options
if known_extensions is None:
known_extensions = KNOWN_EXTENSIONS
self._known_extensions = known_extensions
self._reader_options = reader_options
self._reader_method = reader_method
self._base_directory = base_directory
if boto3_options is None:
boto3_options = {}
# s3fs can read credentials from ~/.aws/credentials, same as boto3
client_kwargs = {}
if boto3_options.get("endpoint_url"):
client_kwargs["endpoint_url"] = boto3_options.get("endpoint_url")
self.fs = s3fs.S3FileSystem(anon=False, client_kwargs=client_kwargs)
def __init__(self, data_context, action_list):
self.data_context = data_context
self.action_list = action_list
self.actions = OrderedDict()
for action_config in action_list:
assert isinstance(action_config, dict)
# NOTE: Eugene: 2019-09-23: need a better way to validate an action config:
if not set(action_config.keys()) == {"name", "action"}:
raise KeyError(
'Action config keys must be ("name", "action"). Instead got {}'.format(action_config.keys())
)
config = action_config["action"]
module_name = 'great_expectations.validation_operators'
new_action = instantiate_class_from_config(
config=config,
runtime_environment={
"data_context": self.data_context,
},
config_defaults={
"module_name": module_name
}
)
if not new_action:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=config['class_name']
)
self.actions[action_config["name"]] = new_action
def __init__(self,
overview_section_renderer=None,
column_section_renderer=None) -> None:
super().__init__()
if overview_section_renderer is None:
overview_section_renderer = {
"class_name": "ProfilingResultsOverviewSectionRenderer"
}
if column_section_renderer is None:
column_section_renderer = {
"class_name": "ProfilingResultsColumnSectionRenderer"
}
module_name = "great_expectations.render.renderer.profiling_results_overview_section_renderer"
self._overview_section_renderer = instantiate_class_from_config(
config=overview_section_renderer,
runtime_environment={},
config_defaults={
"module_name":
overview_section_renderer.get("module_name", module_name)
},
)
if not self._overview_section_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=overview_section_renderer["class_name"],
)
module_name = "great_expectations.render.renderer.column_section_renderer"
self._column_section_renderer = instantiate_class_from_config(
config=column_section_renderer,
runtime_environment={},
config_defaults={
"module_name":
column_section_renderer.get("module_name", module_name)
},
)
if not self._column_section_renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=column_section_renderer["class_name"],
)
def _build_generator(self, **kwargs):
"""Build a generator using the provided configuration and return the newly-built generator."""
module_name = 'great_expectations.datasource.generator'
generator = instantiate_class_from_config(
config=kwargs,
runtime_environment={"datasource": self},
config_defaults={"module_name": module_name})
if not generator:
raise ClassInstantiationError(module_name=module_name,
package_name=None,
class_name=kwargs['class_name'])
return generator
def __init__(
self,
data_context,
renderer,
slack_webhook=None,
slack_token=None,
slack_channel=None,
notify_on="all",
notify_with=None,
):
"""Construct a SlackNotificationAction
Args:
data_context:
renderer: dictionary specifying the renderer used to generate a query consumable by Slack API, for example:
{
"module_name": "great_expectations.render.renderer.slack_renderer",
"class_name": "SlackRenderer",
}
slack_webhook: incoming Slack webhook to which to send notification
notify_on: "all", "failure", "success" - specifies validation status that will trigger notification
payload: *Optional* payload from other ValidationActions
"""
super().__init__(data_context)
self.renderer = instantiate_class_from_config(
config=renderer,
runtime_environment={},
config_defaults={},
)
module_name = renderer["module_name"]
if not self.renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer["class_name"],
)
if not slack_token and slack_channel:
assert slack_webhook
if not slack_webhook:
assert slack_token and slack_channel
assert not (slack_webhook and slack_channel and slack_token)
self.slack_webhook = slack_webhook
self.slack_token = slack_token
self.slack_channel = slack_channel
self.notify_on = notify_on
self.notify_with = notify_with
def __init__(self,
name="default",
datasource=None,
query_store_backend=None,
queries=None):
super().__init__(name=name, datasource=datasource)
if (datasource and datasource.data_context
and datasource.data_context.root_directory):
root_directory = datasource.data_context.root_directory
else:
root_directory = None
if query_store_backend is None:
# We will choose a Tuple store if there is a configured DataContext with a root_directory,
# and an InMemoryStore otherwise
if root_directory:
query_store_backend = {
"class_name":
"TupleFilesystemStoreBackend",
"base_directory":
os.path.join(
datasource.data_context.root_directory,
"datasources",
datasource.name,
"generators",
name,
),
"filepath_suffix":
".sql",
}
else:
query_store_backend = {"class_name": "InMemoryStoreBackend"}
module_name = "great_expectations.data_context.store"
self._store_backend = instantiate_class_from_config(
config=query_store_backend,
runtime_environment={"root_directory": root_directory},
config_defaults={"module_name": module_name},
)
if not self._store_backend:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=query_store_backend["class_name"],
)
if queries is not None:
for query_name, query in queries.items():
self.add_query(data_asset_name=query_name, query=query)
def __init__(
self,
data_context,
action_list,
name,
result_format={"result_format": "SUMMARY"},
):
super().__init__()
self.data_context = data_context
self.name = name
result_format = parse_result_format(result_format)
assert result_format["result_format"] in [
"BOOLEAN_ONLY",
"BASIC",
"SUMMARY",
"COMPLETE",
]
self.result_format = result_format
self.action_list = action_list
self.actions = OrderedDict()
for action_config in action_list:
assert isinstance(action_config, dict)
# NOTE: Eugene: 2019-09-23: need a better way to validate an action config:
if not set(action_config.keys()) == {"name", "action"}:
raise KeyError(
'Action config keys must be ("name", "action"). Instead got {}'.format(
action_config.keys()
)
)
config = action_config["action"]
module_name = "great_expectations.validation_operators"
new_action = instantiate_class_from_config(
config=config,
runtime_environment={"data_context": self.data_context},
config_defaults={"module_name": module_name},
)
if not new_action:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=config["class_name"],
)
self.actions[action_config["name"]] = new_action
def _build_batch_kwargs_generator(self, **kwargs):
"""Build a BatchKwargGenerator using the provided configuration and return the newly-built generator."""
generator = instantiate_class_from_config(
config=kwargs,
runtime_environment={"datasource": self},
config_defaults={
"module_name": "great_expectations.datasource.batch_kwargs_generator"
},
)
if not generator:
raise ClassInstantiationError(
module_name="great_expectations.datasource.batch_kwargs_generator",
package_name=None,
class_name=kwargs["class_name"],
)
return generator
def __init__(self, column_section_renderer=None):
if column_section_renderer is None:
column_section_renderer = {
"class_name": "ExpectationSuiteColumnSectionRenderer"
}
module_name = 'great_expectations.render.renderer.column_section_renderer'
self._column_section_renderer = instantiate_class_from_config(
config=column_section_renderer,
runtime_environment={},
config_defaults={
"module_name":
column_section_renderer.get("module_name", module_name)
})
if not self._column_section_renderer:
raise ClassInstantiationError(
module_name=column_section_renderer,
package_name=None,
class_name=column_section_renderer['class_name'])
def __init__(
self,
data_context,
renderer,
microsoft_teams_webhook,
notify_on="all",
):
"""Construct a MicrosoftTeamsNotificationAction
Args:
data_context:
renderer: dictionary specifying the renderer used to generate a query consumable by teams API, for example:
{
"module_name": "great_expectations.render.renderer.microsoft_teams_renderer",
"class_name": "MicrosoftTeamsRenderer",
}
microsoft_teams_webhook: incoming Microsoft Teams webhook to which to send notifications
notify_on: "all", "failure", "success" - specifies validation status that will trigger notification
payload: *Optional* payload from other ValidationActions
"""
super().__init__(data_context)
self.renderer = instantiate_class_from_config(
config=renderer,
runtime_environment={},
config_defaults={},
)
module_name = renderer["module_name"]
if not self.renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer["class_name"],
)
self.teams_webhook = microsoft_teams_webhook
assert (
microsoft_teams_webhook
), "No Microsoft teams webhook found in action config."
self.notify_on = notify_on
def __init__(
self,
data_context,
renderer,
api_key,
region=None,
priority="P3",
notify_on="failure",
):
"""Construct a OpsgenieAlertAction
Args:
data_context:
api_key: Opsgenie API key
region: specifies the Opsgenie region. Populate 'EU' for Europe otherwise do not set
priority: specify the priority of the alert (P1 - P5) defaults to P3
notify_on: "all", "failure", "success" - specifies validation status that will trigger notification
"""
super().__init__(data_context)
self.renderer = instantiate_class_from_config(
config=renderer,
runtime_environment={},
config_defaults={},
)
module_name = renderer["module_name"]
if not self.renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer["class_name"],
)
self.api_key = api_key
assert api_key, "opsgenie_api_key missing in config_variables.yml"
self.region = region
self.priority = priority
self.notify_on = notify_on
def __init__(self, store_backend=None, runtime_environment=None):
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "TupleFilesystemStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Class was loaded successfully; verify that it is of a correct subclass.
if not issubclass(store_class, TupleStoreBackend):
raise DataContextError(
"Invalid configuration: HtmlSiteStore needs a TupleStoreBackend"
)
if "filepath_template" in store_backend or (
"fixed_length_key" in store_backend
and store_backend["fixed_length_key"] is True):
logger.warning(
"Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: "
"filepaths will be selected based on the type of asset rendered."
)
# One thing to watch for is reversibility of keys.
# If several types are being written to overlapping directories, we could get collisions.
module_name = 'great_expectations.data_context.store'
filepath_prefix = 'expectations'
filepath_suffix = '.html'
expectation_suite_identifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_prefix": filepath_prefix,
"filepath_suffix": filepath_suffix,
})
if not expectation_suite_identifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_prefix = 'validations'
validation_result_idendifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_prefix": filepath_prefix,
"filepath_suffix": filepath_suffix,
})
if not validation_result_idendifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_template = 'index.html'
index_page_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_template": filepath_template,
})
if not index_page_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
filepath_template = None
static_assets_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults={
"module_name": module_name,
"filepath_template": filepath_template,
})
if not static_assets_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend['class_name'])
self.store_backends = {
ExpectationSuiteIdentifier: expectation_suite_identifier_obj,
ValidationResultIdentifier: validation_result_idendifier_obj,
"index_page": index_page_obj,
"static_assets": static_assets_obj,
}
# NOTE: Instead of using the filesystem as the source of record for keys,
# this class tracks keys separately in an internal set.
# This means that keys are stored for a specific session, but can't be fetched after the original
# HtmlSiteStore instance leaves scope.
# Doing it this way allows us to prevent namespace collisions among keys while still having multiple
# backends that write to the same directory structure.
# It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it
# can't necessarily set and list_keys like most other Stores.
self.keys = set()
def __init__(
self,
data_context,
renderer,
smtp_address,
smtp_port,
sender_login,
sender_password,
receiver_emails,
sender_alias=None,
use_tls=None,
use_ssl=None,
notify_on="all",
notify_with=None,
):
"""Construct an EmailAction
Args:
data_context:
renderer: dictionary specifying the renderer used to generate an email, for example:
{
"module_name": "great_expectations.render.renderer.email_renderer",
"class_name": "EmailRenderer",
}
smtp_address: address of the SMTP server used to send the email
smtp_address: port of the SMTP server used to send the email
sender_login: login used send the email
sender_password: password used to send the email
sender_alias: optional alias used to send the email (default = sender_login)
receiver_emails: email addresses that will be receive the email (separated by commas)
use_tls: optional use of TLS to send the email (using either TLS or SSL is highly recommended)
use_ssl: optional use of SSL to send the email (using either TLS or SSL is highly recommended)
notify_on: "all", "failure", "success" - specifies validation status that will trigger notification
notify_with: optional list of DataDocs site names to display in the email message
"""
super().__init__(data_context)
self.renderer = instantiate_class_from_config(
config=renderer,
runtime_environment={},
config_defaults={},
)
module_name = renderer["module_name"]
if not self.renderer:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=renderer["class_name"],
)
self.smtp_address = smtp_address
self.smtp_port = smtp_port
self.sender_login = sender_login
self.sender_password = sender_password
if not sender_alias:
self.sender_alias = sender_login
else:
self.sender_alias = sender_alias
self.receiver_emails_list = list(
map(lambda x: x.strip(), receiver_emails.split(",")))
self.use_tls = use_tls
self.use_ssl = use_ssl
assert smtp_address, "No SMTP server address found in action config."
assert smtp_port, "No SMTP server port found in action config."
assert sender_login, "No login found for sending the email in action config."
assert (sender_password
), "No password found for sending the email in action config."
assert (receiver_emails
), "No email addresses to send the email to in action config."
self.notify_on = notify_on
self.notify_with = notify_with
def __init__(self, store_backend=None, runtime_environment=None):
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "TupleFilesystemStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Class was loaded successfully; verify that it is of a correct subclass.
if not issubclass(store_class,
(TupleStoreBackend, GeCloudStoreBackend)):
raise DataContextError(
"Invalid configuration: HtmlSiteStore needs a TupleStoreBackend or GeCloudStoreBackend"
)
if "filepath_template" in store_backend or (
"fixed_length_key" in store_backend
and store_backend["fixed_length_key"] is True):
logger.warning(
"Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: "
"filepaths will be selected based on the type of asset rendered."
)
# One thing to watch for is reversibility of keys.
# If several types are being written to overlapping directories, we could get collisions.
module_name = "great_expectations.data_context.store"
filepath_suffix = ".html"
is_ge_cloud_store = store_backend[
"class_name"] == "GeCloudStoreBackend"
expectation_config_defaults = {
"module_name": module_name,
"filepath_prefix": "expectations",
"filepath_suffix": filepath_suffix,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
expectation_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
expectation_suite_identifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=expectation_config_defaults,
)
if not expectation_suite_identifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
validation_result_config_defaults = {
"module_name": module_name,
"filepath_prefix": "validations",
"filepath_suffix": filepath_suffix,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
validation_result_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
validation_result_idendifier_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=validation_result_config_defaults,
)
if not validation_result_idendifier_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
filepath_template = "index.html"
index_page_config_defaults = {
"module_name": module_name,
"filepath_template": filepath_template,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
index_page_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
index_page_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=index_page_config_defaults,
)
if not index_page_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
static_assets_config_defaults = {
"module_name": module_name,
"filepath_template": None,
"suppress_store_backend_id": True,
}
if is_ge_cloud_store:
static_assets_config_defaults = {
"module_name": module_name,
"suppress_store_backend_id": True,
}
static_assets_obj = instantiate_class_from_config(
config=store_backend,
runtime_environment=runtime_environment,
config_defaults=static_assets_config_defaults,
)
if not static_assets_obj:
raise ClassInstantiationError(
module_name=module_name,
package_name=None,
class_name=store_backend["class_name"],
)
self.store_backends = {
ExpectationSuiteIdentifier: expectation_suite_identifier_obj,
ValidationResultIdentifier: validation_result_idendifier_obj,
"index_page": index_page_obj,
"static_assets": static_assets_obj,
}
# NOTE: Instead of using the filesystem as the source of record for keys,
# this class tracks keys separately in an internal set.
# This means that keys are stored for a specific session, but can't be fetched after the original
# HtmlSiteStore instance leaves scope.
# Doing it this way allows us to prevent namespace collisions among keys while still having multiple
# backends that write to the same directory structure.
# It's a pretty reasonable way for HtmlSiteStore to do its job---you just have to remember that it
# can't necessarily set and list_keys like most other Stores.
self.keys = set()
# Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter
# out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary.
self._config = {
"store_backend": store_backend,
"runtime_environment": runtime_environment,
"module_name": self.__class__.__module__,
"class_name": self.__class__.__name__,
}
filter_properties_dict(properties=self._config,
clean_falsy=True,
inplace=True)
