def __init__( self, overview_table_renderer=None, expectation_string_renderer=None, runtime_environment=None, ): super().__init__() if overview_table_renderer is None: overview_table_renderer = { "class_name": "ProfilingOverviewTableContentBlockRenderer" } if expectation_string_renderer is None: expectation_string_renderer = { "class_name": "ExpectationStringRenderer" } module_name = "great_expectations.render.renderer.content_block" self._overview_table_renderer = instantiate_class_from_config( config=overview_table_renderer, runtime_environment=runtime_environment, config_defaults={"module_name": module_name}, ) if not self._overview_table_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=overview_table_renderer["class_name"], ) self._expectation_string_renderer = instantiate_class_from_config( config=expectation_string_renderer, runtime_environment=runtime_environment, config_defaults={"module_name": module_name}, ) if not self._expectation_string_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=expectation_string_renderer["class_name"], ) self.content_block_function_names = [ "_render_header", "_render_overview_table", "_render_quantile_table", "_render_stats_table", "_render_values_set", "_render_histogram", "_render_bar_chart_table", "_render_failed", ]
def __init__(self, properties_table_renderer=None, runtime_environment=None): super().__init__() if properties_table_renderer is None: properties_table_renderer = { "class_name": "ProfilingColumnPropertiesTableContentBlockRenderer" } module_name = "great_expectations.render.renderer.content_block" self._properties_table_renderer = instantiate_class_from_config( config=properties_table_renderer, runtime_environment=runtime_environment, config_defaults={"module_name": module_name}, ) if not self._properties_table_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=properties_table_renderer["class_name"], ) self.content_block_function_names = [ "_render_header", "_render_properties_table", "_render_quantile_table", "_render_stats_table", "_render_values_set", "_render_histogram", "_render_value_counts_bar_chart", "_render_failed", ]
def render(self) -> None: """Renders content using the: - atomic prescriptive renderer for the expectation configuration associated with this ExpectationValidationResult to self.expectation_config.rendered_content - atomic diagnostic renderer for the expectation configuration associated with this ExpectationValidationResult to self.rendered_content. """ inline_renderer_config: Dict[str, Union[str, ExpectationValidationResult]] = { "class_name": "InlineRenderer", "render_object": self, } module_name: str = "great_expectations.render.renderer.inline_renderer" inline_renderer = instantiate_class_from_config( config=inline_renderer_config, runtime_environment={}, config_defaults={"module_name": module_name}, ) if not inline_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=inline_renderer_config["class_name"], ) ( self.expectation_config.rendered_content, self.rendered_content, ) = inline_renderer.render()
def __init__(self, store_backend=None, runtime_environment=None, store_name="no_store_name"): """ Runtime environment may be necessary to instantiate store backend elements. Args: store_backend: runtime_environment: store_name: store name given in the DataContextConfig (via either in-code or yaml configuration) """ if store_backend is None: store_backend = {"class_name": "InMemoryStoreBackend"} self._store_name = store_name logger.debug("Building store_backend.") module_name = "great_expectations.data_context.store" self._store_backend = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment or {}, config_defaults={ "module_name": module_name, "store_name": self._store_name, }, ) if not self._store_backend: raise ClassInstantiationError(module_name=module_name, package_name=None, class_name=store_backend) if not isinstance(self._store_backend, StoreBackend): raise DataContextError( "Invalid StoreBackend configuration: expected a StoreBackend instance." ) self._use_fixed_length_key = self._store_backend.fixed_length_key
def __init__( self, column_section_renderer=None, run_info_at_end: bool = False, data_context=None, ) -> None: """ Args: column_section_renderer: run_info_at_end: Move the run info (Info, Batch Markers, Batch Kwargs) to the end of the rendered output rather than after Statistics. """ super().__init__() if column_section_renderer is None: column_section_renderer = { "class_name": "ValidationResultsColumnSectionRenderer" } module_name = "great_expectations.render.renderer.column_section_renderer" self._column_section_renderer = instantiate_class_from_config( config=column_section_renderer, runtime_environment={}, config_defaults={ "module_name": column_section_renderer.get("module_name", module_name) }, ) if not self._column_section_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=column_section_renderer["class_name"], ) self.run_info_at_end = run_info_at_end self._data_context = data_context
def __init__( self, name="default", datasource=None, bucket=None, boto3_options=None, base_directory="/data", reader_options=None, known_extensions=None, reader_method=None, ): super().__init__(name, datasource=datasource) if not s3fs: raise ClassInstantiationError( "ModuleNotFoundError: No module named 's3fs'") if reader_options is None: reader_options = self._default_reader_options if known_extensions is None: known_extensions = KNOWN_EXTENSIONS self._known_extensions = known_extensions self._reader_options = reader_options self._reader_method = reader_method self._base_directory = base_directory if boto3_options is None: boto3_options = {} # s3fs can read credentials from ~/.aws/credentials, same as boto3 client_kwargs = {} if boto3_options.get("endpoint_url"): client_kwargs["endpoint_url"] = boto3_options.get("endpoint_url") self.fs = s3fs.S3FileSystem(anon=False, client_kwargs=client_kwargs)
def __init__(self, data_context, action_list): self.data_context = data_context self.action_list = action_list self.actions = OrderedDict() for action_config in action_list: assert isinstance(action_config, dict) # NOTE: Eugene: 2019-09-23: need a better way to validate an action config: if not set(action_config.keys()) == {"name", "action"}: raise KeyError( 'Action config keys must be ("name", "action"). Instead got {}'.format(action_config.keys()) ) config = action_config["action"] module_name = 'great_expectations.validation_operators' new_action = instantiate_class_from_config( config=config, runtime_environment={ "data_context": self.data_context, }, config_defaults={ "module_name": module_name } ) if not new_action: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=config['class_name'] ) self.actions[action_config["name"]] = new_action
def __init__(self, overview_section_renderer=None, column_section_renderer=None) -> None: super().__init__() if overview_section_renderer is None: overview_section_renderer = { "class_name": "ProfilingResultsOverviewSectionRenderer" } if column_section_renderer is None: column_section_renderer = { "class_name": "ProfilingResultsColumnSectionRenderer" } module_name = "great_expectations.render.renderer.profiling_results_overview_section_renderer" self._overview_section_renderer = instantiate_class_from_config( config=overview_section_renderer, runtime_environment={}, config_defaults={ "module_name": overview_section_renderer.get("module_name", module_name) }, ) if not self._overview_section_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=overview_section_renderer["class_name"], ) module_name = "great_expectations.render.renderer.column_section_renderer" self._column_section_renderer = instantiate_class_from_config( config=column_section_renderer, runtime_environment={}, config_defaults={ "module_name": column_section_renderer.get("module_name", module_name) }, ) if not self._column_section_renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=column_section_renderer["class_name"], )
def _build_generator(self, **kwargs): """Build a generator using the provided configuration and return the newly-built generator.""" module_name = 'great_expectations.datasource.generator' generator = instantiate_class_from_config( config=kwargs, runtime_environment={"datasource": self}, config_defaults={"module_name": module_name}) if not generator: raise ClassInstantiationError(module_name=module_name, package_name=None, class_name=kwargs['class_name']) return generator
def __init__( self, data_context, renderer, slack_webhook=None, slack_token=None, slack_channel=None, notify_on="all", notify_with=None, ): """Construct a SlackNotificationAction Args: data_context: renderer: dictionary specifying the renderer used to generate a query consumable by Slack API, for example: { "module_name": "great_expectations.render.renderer.slack_renderer", "class_name": "SlackRenderer", } slack_webhook: incoming Slack webhook to which to send notification notify_on: "all", "failure", "success" - specifies validation status that will trigger notification payload: *Optional* payload from other ValidationActions """ super().__init__(data_context) self.renderer = instantiate_class_from_config( config=renderer, runtime_environment={}, config_defaults={}, ) module_name = renderer["module_name"] if not self.renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=renderer["class_name"], ) if not slack_token and slack_channel: assert slack_webhook if not slack_webhook: assert slack_token and slack_channel assert not (slack_webhook and slack_channel and slack_token) self.slack_webhook = slack_webhook self.slack_token = slack_token self.slack_channel = slack_channel self.notify_on = notify_on self.notify_with = notify_with
def __init__(self, name="default", datasource=None, query_store_backend=None, queries=None): super().__init__(name=name, datasource=datasource) if (datasource and datasource.data_context and datasource.data_context.root_directory): root_directory = datasource.data_context.root_directory else: root_directory = None if query_store_backend is None: # We will choose a Tuple store if there is a configured DataContext with a root_directory, # and an InMemoryStore otherwise if root_directory: query_store_backend = { "class_name": "TupleFilesystemStoreBackend", "base_directory": os.path.join( datasource.data_context.root_directory, "datasources", datasource.name, "generators", name, ), "filepath_suffix": ".sql", } else: query_store_backend = {"class_name": "InMemoryStoreBackend"} module_name = "great_expectations.data_context.store" self._store_backend = instantiate_class_from_config( config=query_store_backend, runtime_environment={"root_directory": root_directory}, config_defaults={"module_name": module_name}, ) if not self._store_backend: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=query_store_backend["class_name"], ) if queries is not None: for query_name, query in queries.items(): self.add_query(data_asset_name=query_name, query=query)
def __init__( self, data_context, action_list, name, result_format={"result_format": "SUMMARY"}, ): super().__init__() self.data_context = data_context self.name = name result_format = parse_result_format(result_format) assert result_format["result_format"] in [ "BOOLEAN_ONLY", "BASIC", "SUMMARY", "COMPLETE", ] self.result_format = result_format self.action_list = action_list self.actions = OrderedDict() for action_config in action_list: assert isinstance(action_config, dict) # NOTE: Eugene: 2019-09-23: need a better way to validate an action config: if not set(action_config.keys()) == {"name", "action"}: raise KeyError( 'Action config keys must be ("name", "action"). Instead got {}'.format( action_config.keys() ) ) config = action_config["action"] module_name = "great_expectations.validation_operators" new_action = instantiate_class_from_config( config=config, runtime_environment={"data_context": self.data_context}, config_defaults={"module_name": module_name}, ) if not new_action: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=config["class_name"], ) self.actions[action_config["name"]] = new_action
def _build_batch_kwargs_generator(self, **kwargs): """Build a BatchKwargGenerator using the provided configuration and return the newly-built generator.""" generator = instantiate_class_from_config( config=kwargs, runtime_environment={"datasource": self}, config_defaults={ "module_name": "great_expectations.datasource.batch_kwargs_generator" }, ) if not generator: raise ClassInstantiationError( module_name="great_expectations.datasource.batch_kwargs_generator", package_name=None, class_name=kwargs["class_name"], ) return generator
def __init__(self, column_section_renderer=None): if column_section_renderer is None: column_section_renderer = { "class_name": "ExpectationSuiteColumnSectionRenderer" } module_name = 'great_expectations.render.renderer.column_section_renderer' self._column_section_renderer = instantiate_class_from_config( config=column_section_renderer, runtime_environment={}, config_defaults={ "module_name": column_section_renderer.get("module_name", module_name) }) if not self._column_section_renderer: raise ClassInstantiationError( module_name=column_section_renderer, package_name=None, class_name=column_section_renderer['class_name'])
def __init__( self, data_context, renderer, microsoft_teams_webhook, notify_on="all", ): """Construct a MicrosoftTeamsNotificationAction Args: data_context: renderer: dictionary specifying the renderer used to generate a query consumable by teams API, for example: { "module_name": "great_expectations.render.renderer.microsoft_teams_renderer", "class_name": "MicrosoftTeamsRenderer", } microsoft_teams_webhook: incoming Microsoft Teams webhook to which to send notifications notify_on: "all", "failure", "success" - specifies validation status that will trigger notification payload: *Optional* payload from other ValidationActions """ super().__init__(data_context) self.renderer = instantiate_class_from_config( config=renderer, runtime_environment={}, config_defaults={}, ) module_name = renderer["module_name"] if not self.renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=renderer["class_name"], ) self.teams_webhook = microsoft_teams_webhook assert ( microsoft_teams_webhook ), "No Microsoft teams webhook found in action config." self.notify_on = notify_on
def __init__( self, data_context, renderer, api_key, region=None, priority="P3", notify_on="failure", ): """Construct a OpsgenieAlertAction Args: data_context: api_key: Opsgenie API key region: specifies the Opsgenie region. Populate 'EU' for Europe otherwise do not set priority: specify the priority of the alert (P1 - P5) defaults to P3 notify_on: "all", "failure", "success" - specifies validation status that will trigger notification """ super().__init__(data_context) self.renderer = instantiate_class_from_config( config=renderer, runtime_environment={}, config_defaults={}, ) module_name = renderer["module_name"] if not self.renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=renderer["class_name"], ) self.api_key = api_key assert api_key, "opsgenie_api_key missing in config_variables.yml" self.region = region self.priority = priority self.notify_on = notify_on
def __init__(self, store_backend=None, runtime_environment=None): store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "TupleFilesystemStoreBackend") verify_dynamic_loading_support(module_name=store_backend_module_name) store_class = load_class(store_backend_class_name, store_backend_module_name) # Store Class was loaded successfully; verify that it is of a correct subclass. if not issubclass(store_class, TupleStoreBackend): raise DataContextError( "Invalid configuration: HtmlSiteStore needs a TupleStoreBackend" ) if "filepath_template" in store_backend or ( "fixed_length_key" in store_backend and store_backend["fixed_length_key"] is True): logger.warning( "Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: " "filepaths will be selected based on the type of asset rendered." ) # One thing to watch for is reversibility of keys. # If several types are being written to overlapping directories, we could get collisions. module_name = 'great_expectations.data_context.store' filepath_prefix = 'expectations' filepath_suffix = '.html' expectation_suite_identifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_prefix": filepath_prefix, "filepath_suffix": filepath_suffix, }) if not expectation_suite_identifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_prefix = 'validations' validation_result_idendifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_prefix": filepath_prefix, "filepath_suffix": filepath_suffix, }) if not validation_result_idendifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_template = 'index.html' index_page_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_template": filepath_template, }) if not index_page_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) filepath_template = None static_assets_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults={ "module_name": module_name, "filepath_template": filepath_template, }) if not static_assets_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend['class_name']) self.store_backends = { ExpectationSuiteIdentifier: expectation_suite_identifier_obj, ValidationResultIdentifier: validation_result_idendifier_obj, "index_page": index_page_obj, "static_assets": static_assets_obj, } # NOTE: Instead of using the filesystem as the source of record for keys, # this class tracks keys separately in an internal set. # This means that keys are stored for a specific session, but can't be fetched after the original # HtmlSiteStore instance leaves scope. # Doing it this way allows us to prevent namespace collisions among keys while still having multiple # backends that write to the same directory structure. # It's a pretty reasonable way for HtmlSiteStore to do its job---you just ahve to remember that it # can't necessarily set and list_keys like most other Stores. self.keys = set()
def __init__( self, data_context, renderer, smtp_address, smtp_port, sender_login, sender_password, receiver_emails, sender_alias=None, use_tls=None, use_ssl=None, notify_on="all", notify_with=None, ): """Construct an EmailAction Args: data_context: renderer: dictionary specifying the renderer used to generate an email, for example: { "module_name": "great_expectations.render.renderer.email_renderer", "class_name": "EmailRenderer", } smtp_address: address of the SMTP server used to send the email smtp_address: port of the SMTP server used to send the email sender_login: login used send the email sender_password: password used to send the email sender_alias: optional alias used to send the email (default = sender_login) receiver_emails: email addresses that will be receive the email (separated by commas) use_tls: optional use of TLS to send the email (using either TLS or SSL is highly recommended) use_ssl: optional use of SSL to send the email (using either TLS or SSL is highly recommended) notify_on: "all", "failure", "success" - specifies validation status that will trigger notification notify_with: optional list of DataDocs site names to display in the email message """ super().__init__(data_context) self.renderer = instantiate_class_from_config( config=renderer, runtime_environment={}, config_defaults={}, ) module_name = renderer["module_name"] if not self.renderer: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=renderer["class_name"], ) self.smtp_address = smtp_address self.smtp_port = smtp_port self.sender_login = sender_login self.sender_password = sender_password if not sender_alias: self.sender_alias = sender_login else: self.sender_alias = sender_alias self.receiver_emails_list = list( map(lambda x: x.strip(), receiver_emails.split(","))) self.use_tls = use_tls self.use_ssl = use_ssl assert smtp_address, "No SMTP server address found in action config." assert smtp_port, "No SMTP server port found in action config." assert sender_login, "No login found for sending the email in action config." assert (sender_password ), "No password found for sending the email in action config." assert (receiver_emails ), "No email addresses to send the email to in action config." self.notify_on = notify_on self.notify_with = notify_with
def __init__(self, store_backend=None, runtime_environment=None): store_backend_module_name = store_backend.get( "module_name", "great_expectations.data_context.store") store_backend_class_name = store_backend.get( "class_name", "TupleFilesystemStoreBackend") verify_dynamic_loading_support(module_name=store_backend_module_name) store_class = load_class(store_backend_class_name, store_backend_module_name) # Store Class was loaded successfully; verify that it is of a correct subclass. if not issubclass(store_class, (TupleStoreBackend, GeCloudStoreBackend)): raise DataContextError( "Invalid configuration: HtmlSiteStore needs a TupleStoreBackend or GeCloudStoreBackend" ) if "filepath_template" in store_backend or ( "fixed_length_key" in store_backend and store_backend["fixed_length_key"] is True): logger.warning( "Configuring a filepath_template or using fixed_length_key is not supported in SiteBuilder: " "filepaths will be selected based on the type of asset rendered." ) # One thing to watch for is reversibility of keys. # If several types are being written to overlapping directories, we could get collisions. module_name = "great_expectations.data_context.store" filepath_suffix = ".html" is_ge_cloud_store = store_backend[ "class_name"] == "GeCloudStoreBackend" expectation_config_defaults = { "module_name": module_name, "filepath_prefix": "expectations", "filepath_suffix": filepath_suffix, "suppress_store_backend_id": True, } if is_ge_cloud_store: expectation_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } expectation_suite_identifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=expectation_config_defaults, ) if not expectation_suite_identifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) validation_result_config_defaults = { "module_name": module_name, "filepath_prefix": "validations", "filepath_suffix": filepath_suffix, "suppress_store_backend_id": True, } if is_ge_cloud_store: validation_result_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } validation_result_idendifier_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=validation_result_config_defaults, ) if not validation_result_idendifier_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) filepath_template = "index.html" index_page_config_defaults = { "module_name": module_name, "filepath_template": filepath_template, "suppress_store_backend_id": True, } if is_ge_cloud_store: index_page_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } index_page_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=index_page_config_defaults, ) if not index_page_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) static_assets_config_defaults = { "module_name": module_name, "filepath_template": None, "suppress_store_backend_id": True, } if is_ge_cloud_store: static_assets_config_defaults = { "module_name": module_name, "suppress_store_backend_id": True, } static_assets_obj = instantiate_class_from_config( config=store_backend, runtime_environment=runtime_environment, config_defaults=static_assets_config_defaults, ) if not static_assets_obj: raise ClassInstantiationError( module_name=module_name, package_name=None, class_name=store_backend["class_name"], ) self.store_backends = { ExpectationSuiteIdentifier: expectation_suite_identifier_obj, ValidationResultIdentifier: validation_result_idendifier_obj, "index_page": index_page_obj, "static_assets": static_assets_obj, } # NOTE: Instead of using the filesystem as the source of record for keys, # this class tracks keys separately in an internal set. # This means that keys are stored for a specific session, but can't be fetched after the original # HtmlSiteStore instance leaves scope. # Doing it this way allows us to prevent namespace collisions among keys while still having multiple # backends that write to the same directory structure. # It's a pretty reasonable way for HtmlSiteStore to do its job---you just have to remember that it # can't necessarily set and list_keys like most other Stores. self.keys = set() # Gather the call arguments of the present function (include the "module_name" and add the "class_name"), filter # out the Falsy values, and set the instance "_config" variable equal to the resulting dictionary. self._config = { "store_backend": store_backend, "runtime_environment": runtime_environment, "module_name": self.__class__.__module__, "class_name": self.__class__.__name__, } filter_properties_dict(properties=self._config, clean_falsy=True, inplace=True)