def build(self, resource_identifiers=None):
source_store_keys = self.source_store.list_keys()
if self.name == "validations" and self.validation_results_limit:
source_store_keys = sorted(source_store_keys, key=lambda x: x.run_id, reverse=True)[:self.validation_results_limit]
for resource_key in source_store_keys:
# if no resource_identifiers are passed, the section builder will build
# a page for every keys in its source store.
# if the caller did pass resource_identifiers, the section builder
# will build pages only for the specified resources
if resource_identifiers and resource_key not in resource_identifiers:
continue
if self.run_id_filter:
if not self._resource_key_passes_run_id_filter(resource_key):
continue
resource = self.source_store.get(resource_key)
if isinstance(resource_key, ExpectationSuiteIdentifier):
expectation_suite_name = resource_key.expectation_suite_name
logger.debug(" Rendering expectation suite {}".format(expectation_suite_name))
elif isinstance(resource_key, ValidationResultIdentifier):
run_id = resource_key.run_id
expectation_suite_name = resource_key.expectation_suite_identifier.expectation_suite_name
if run_id == "profiling":
logger.debug(" Rendering profiling for batch {}".format(resource_key.batch_identifier))
else:
logger.debug(
" Rendering validation: run id: {}, suite {} for batch {}".format(
run_id,
expectation_suite_name,
resource_key.batch_identifier
)
)
try:
rendered_content = self.renderer_class.render(resource)
viewable_content = self.view_class.render(rendered_content)
except Exception as e:
exception_message = f'''\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
'''
exception_traceback = traceback.format_exc()
exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
logger.error(exception_message, e, exc_info=True)
self.target_store.set(
SiteSectionIdentifier(
site_section_name=self.name,
resource_identifier=resource_key,
),
viewable_content
)
def build(self, resource_identifiers=None):
source_store_keys = self.source_store.list_keys()
if self.name == "validations" and self.validation_results_limit:
source_store_keys = sorted(
source_store_keys, key=lambda x: x.run_id,
reverse=True)[:self.validation_results_limit]
for resource_key in source_store_keys:
# if no resource_identifiers are passed, the section builder will build
# a page for every keys in its source store.
# if the caller did pass resource_identifiers, the section builder
# will build pages only for the specified resources
if resource_identifiers and resource_key not in resource_identifiers:
continue
if self.run_id_filter:
if not self._resource_key_passes_run_id_filter(resource_key):
continue
resource = self.source_store.get(resource_key)
if isinstance(resource_key, ExpectationSuiteIdentifier):
expectation_suite_name = resource_key.expectation_suite_name
logger.debug(" Rendering expectation suite {}".format(
expectation_suite_name))
elif isinstance(resource_key, ValidationResultIdentifier):
run_id = resource_key.run_id
expectation_suite_name = resource_key.expectation_suite_identifier.expectation_suite_name
if run_id == "profiling":
logger.debug(
" Rendering profiling for batch {}".format(
resource_key.batch_identifier))
else:
logger.debug(
" Rendering validation: run id: {}, suite {} for batch {}"
.format(run_id, expectation_suite_name,
resource_key.batch_identifier))
try:
rendered_content = self.renderer_class.render(resource)
viewable_content = self.view_class.render(rendered_content)
except Exception as e:
logger.error("Exception occurred during data docs rendering: ",
e,
exc_info=True)
continue
self.target_store.set(
SiteSectionIdentifier(
site_section_name=self.name,
resource_identifier=resource_key,
), viewable_content)
def build(self, resource_identifiers=None) -> None:
source_store_keys = self.source_store.list_keys()
if self.name == "validations" and self.validation_results_limit:
source_store_keys = sorted(
source_store_keys,
key=lambda x: x.run_id.run_time,
reverse=True)[:self.validation_results_limit]
for resource_key in source_store_keys:
# if no resource_identifiers are passed, the section
# builder will build
# a page for every keys in its source store.
# if the caller did pass resource_identifiers, the section builder
# will build pages only for the specified resources
if resource_identifiers and resource_key not in resource_identifiers:
continue
if self.run_name_filter and not isinstance(resource_key,
GeCloudIdentifier):
if not resource_key_passes_run_name_filter(
resource_key, self.run_name_filter):
continue
try:
resource = self.source_store.get(resource_key)
if isinstance(resource_key, ExpectationSuiteIdentifier):
resource = ExpectationSuite(**resource,
data_context=self.data_context)
except exceptions.InvalidKeyError:
logger.warning(
f"Object with Key: {str(resource_key)} could not be retrieved. Skipping..."
)
continue
if isinstance(resource_key, ExpectationSuiteIdentifier):
expectation_suite_name = resource_key.expectation_suite_name
logger.debug(
f" Rendering expectation suite {expectation_suite_name}"
)
elif isinstance(resource_key, ValidationResultIdentifier):
run_id = resource_key.run_id
run_name = run_id.run_name
run_time = run_id.run_time
expectation_suite_name = (
resource_key.expectation_suite_identifier.
expectation_suite_name)
if self.name == "profiling":
logger.debug(
f" Rendering profiling for batch {resource_key.batch_identifier}"
)
else:
logger.debug(
f" Rendering validation: run name: {run_name}, run time: {run_time}, suite {expectation_suite_name} for batch {resource_key.batch_identifier}"
)
try:
rendered_content = self.renderer_class.render(resource)
if self.ge_cloud_mode:
self.target_store.set(
GeCloudIdentifier(resource_type=GeCloudRESTResource.
RENDERED_DATA_DOC),
rendered_content,
source_type=resource_key.resource_type,
source_id=resource_key.ge_cloud_id,
)
else:
viewable_content = self.view_class.render(
rendered_content,
data_context_id=self.data_context_id,
show_how_to_buttons=self.show_how_to_buttons,
)
# Verify type
self.target_store.set(
SiteSectionIdentifier(
site_section_name=self.name,
resource_identifier=resource_key,
),
viewable_content,
)
except Exception as e:
exception_message = """\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
"""
exception_traceback = traceback.format_exc()
exception_message += (f'{type(e).__name__}: "{str(e)}". '
f'Traceback: "{exception_traceback}".')
logger.error(exception_message)
def build(self, resource_identifiers=None):
source_store_keys = self.source_store.list_keys()
if self.name == "validations" and self.validation_results_limit:
source_store_keys = sorted(
source_store_keys,
key=lambda x: x.run_id.run_time,
reverse=True)[:self.validation_results_limit]
expectation_suite_identifier_exists: bool = any([
isinstance(ri, ExpectationSuiteIdentifier)
for ri in resource_identifiers
]) if resource_identifiers is not None else False
for resource_key in source_store_keys:
# All expectation suites are always rendered unless resource_identifiers contains ExpectationSuiteIdentifier(s).
if expectation_suite_identifier_exists or (self.name !=
"expectations"):
# if no resource_identifiers are passed, the section
# builder will build
# a page for every keys in its source store.
# if the caller did pass resource_identifiers, the section builder
# will build pages only for the specified resources
if resource_identifiers and resource_key not in resource_identifiers:
continue
if self.run_name_filter:
if not resource_key_passes_run_name_filter(
resource_key, self.run_name_filter):
continue
try:
resource = self.source_store.get(resource_key)
except exceptions.InvalidKeyError:
logger.warning(
f"Object with Key: {str(resource_key)} could not be retrieved. Skipping..."
)
continue
if isinstance(resource_key, ExpectationSuiteIdentifier):
expectation_suite_name = resource_key.expectation_suite_name
logger.debug(" Rendering expectation suite {}".format(
expectation_suite_name))
elif isinstance(resource_key, ValidationResultIdentifier):
run_id = resource_key.run_id
run_name = run_id.run_name
run_time = run_id.run_time
expectation_suite_name = (
resource_key.expectation_suite_identifier.
expectation_suite_name)
if self.name == "profiling":
logger.debug(
" Rendering profiling for batch {}".format(
resource_key.batch_identifier))
else:
logger.debug(
" Rendering validation: run name: {}, run time: {}, suite {} for batch {}"
.format(
run_name,
run_time,
expectation_suite_name,
resource_key.batch_identifier,
))
try:
rendered_content = self.renderer_class.render(resource)
viewable_content = self.view_class.render(
rendered_content,
data_context_id=self.data_context_id,
show_how_to_buttons=self.show_how_to_buttons,
)
self.target_store.set(
SiteSectionIdentifier(
site_section_name=self.name,
resource_identifier=resource_key,
),
viewable_content,
)
except Exception as e:
exception_message = f"""\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
"""
exception_traceback = traceback.format_exc()
exception_message += (f'{type(e).__name__}: "{str(e)}". '
f'Traceback: "{exception_traceback}".')
logger.error(exception_message, e, exc_info=True)
