def build(self, resource_identifiers=None): source_store_keys = self.source_store.list_keys() if self.name == "validations" and self.validation_results_limit: source_store_keys = sorted(source_store_keys, key=lambda x: x.run_id, reverse=True)[:self.validation_results_limit] for resource_key in source_store_keys: # if no resource_identifiers are passed, the section builder will build # a page for every keys in its source store. # if the caller did pass resource_identifiers, the section builder # will build pages only for the specified resources if resource_identifiers and resource_key not in resource_identifiers: continue if self.run_id_filter: if not self._resource_key_passes_run_id_filter(resource_key): continue resource = self.source_store.get(resource_key) if isinstance(resource_key, ExpectationSuiteIdentifier): expectation_suite_name = resource_key.expectation_suite_name logger.debug(" Rendering expectation suite {}".format(expectation_suite_name)) elif isinstance(resource_key, ValidationResultIdentifier): run_id = resource_key.run_id expectation_suite_name = resource_key.expectation_suite_identifier.expectation_suite_name if run_id == "profiling": logger.debug(" Rendering profiling for batch {}".format(resource_key.batch_identifier)) else: logger.debug( " Rendering validation: run id: {}, suite {} for batch {}".format( run_id, expectation_suite_name, resource_key.batch_identifier ) ) try: rendered_content = self.renderer_class.render(resource) viewable_content = self.view_class.render(rendered_content) except Exception as e: exception_message = f'''\ An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \ not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \ diagnose and repair the underlying issue. Detailed information follows: ''' exception_traceback = traceback.format_exc() exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' logger.error(exception_message, e, exc_info=True) self.target_store.set( SiteSectionIdentifier( site_section_name=self.name, resource_identifier=resource_key, ), viewable_content )
def build(self, resource_identifiers=None): source_store_keys = self.source_store.list_keys() if self.name == "validations" and self.validation_results_limit: source_store_keys = sorted( source_store_keys, key=lambda x: x.run_id, reverse=True)[:self.validation_results_limit] for resource_key in source_store_keys: # if no resource_identifiers are passed, the section builder will build # a page for every keys in its source store. # if the caller did pass resource_identifiers, the section builder # will build pages only for the specified resources if resource_identifiers and resource_key not in resource_identifiers: continue if self.run_id_filter: if not self._resource_key_passes_run_id_filter(resource_key): continue resource = self.source_store.get(resource_key) if isinstance(resource_key, ExpectationSuiteIdentifier): expectation_suite_name = resource_key.expectation_suite_name logger.debug(" Rendering expectation suite {}".format( expectation_suite_name)) elif isinstance(resource_key, ValidationResultIdentifier): run_id = resource_key.run_id expectation_suite_name = resource_key.expectation_suite_identifier.expectation_suite_name if run_id == "profiling": logger.debug( " Rendering profiling for batch {}".format( resource_key.batch_identifier)) else: logger.debug( " Rendering validation: run id: {}, suite {} for batch {}" .format(run_id, expectation_suite_name, resource_key.batch_identifier)) try: rendered_content = self.renderer_class.render(resource) viewable_content = self.view_class.render(rendered_content) except Exception as e: logger.error("Exception occurred during data docs rendering: ", e, exc_info=True) continue self.target_store.set( SiteSectionIdentifier( site_section_name=self.name, resource_identifier=resource_key, ), viewable_content)
def build(self, resource_identifiers=None) -> None: source_store_keys = self.source_store.list_keys() if self.name == "validations" and self.validation_results_limit: source_store_keys = sorted( source_store_keys, key=lambda x: x.run_id.run_time, reverse=True)[:self.validation_results_limit] for resource_key in source_store_keys: # if no resource_identifiers are passed, the section # builder will build # a page for every keys in its source store. # if the caller did pass resource_identifiers, the section builder # will build pages only for the specified resources if resource_identifiers and resource_key not in resource_identifiers: continue if self.run_name_filter and not isinstance(resource_key, GeCloudIdentifier): if not resource_key_passes_run_name_filter( resource_key, self.run_name_filter): continue try: resource = self.source_store.get(resource_key) if isinstance(resource_key, ExpectationSuiteIdentifier): resource = ExpectationSuite(**resource, data_context=self.data_context) except exceptions.InvalidKeyError: logger.warning( f"Object with Key: {str(resource_key)} could not be retrieved. Skipping..." ) continue if isinstance(resource_key, ExpectationSuiteIdentifier): expectation_suite_name = resource_key.expectation_suite_name logger.debug( f" Rendering expectation suite {expectation_suite_name}" ) elif isinstance(resource_key, ValidationResultIdentifier): run_id = resource_key.run_id run_name = run_id.run_name run_time = run_id.run_time expectation_suite_name = ( resource_key.expectation_suite_identifier. expectation_suite_name) if self.name == "profiling": logger.debug( f" Rendering profiling for batch {resource_key.batch_identifier}" ) else: logger.debug( f" Rendering validation: run name: {run_name}, run time: {run_time}, suite {expectation_suite_name} for batch {resource_key.batch_identifier}" ) try: rendered_content = self.renderer_class.render(resource) if self.ge_cloud_mode: self.target_store.set( GeCloudIdentifier(resource_type=GeCloudRESTResource. RENDERED_DATA_DOC), rendered_content, source_type=resource_key.resource_type, source_id=resource_key.ge_cloud_id, ) else: viewable_content = self.view_class.render( rendered_content, data_context_id=self.data_context_id, show_how_to_buttons=self.show_how_to_buttons, ) # Verify type self.target_store.set( SiteSectionIdentifier( site_section_name=self.name, resource_identifier=resource_key, ), viewable_content, ) except Exception as e: exception_message = """\ An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \ not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \ diagnose and repair the underlying issue. Detailed information follows: """ exception_traceback = traceback.format_exc() exception_message += (f'{type(e).__name__}: "{str(e)}". ' f'Traceback: "{exception_traceback}".') logger.error(exception_message)
def build(self, resource_identifiers=None): source_store_keys = self.source_store.list_keys() if self.name == "validations" and self.validation_results_limit: source_store_keys = sorted( source_store_keys, key=lambda x: x.run_id.run_time, reverse=True)[:self.validation_results_limit] expectation_suite_identifier_exists: bool = any([ isinstance(ri, ExpectationSuiteIdentifier) for ri in resource_identifiers ]) if resource_identifiers is not None else False for resource_key in source_store_keys: # All expectation suites are always rendered unless resource_identifiers contains ExpectationSuiteIdentifier(s). if expectation_suite_identifier_exists or (self.name != "expectations"): # if no resource_identifiers are passed, the section # builder will build # a page for every keys in its source store. # if the caller did pass resource_identifiers, the section builder # will build pages only for the specified resources if resource_identifiers and resource_key not in resource_identifiers: continue if self.run_name_filter: if not resource_key_passes_run_name_filter( resource_key, self.run_name_filter): continue try: resource = self.source_store.get(resource_key) except exceptions.InvalidKeyError: logger.warning( f"Object with Key: {str(resource_key)} could not be retrieved. Skipping..." ) continue if isinstance(resource_key, ExpectationSuiteIdentifier): expectation_suite_name = resource_key.expectation_suite_name logger.debug(" Rendering expectation suite {}".format( expectation_suite_name)) elif isinstance(resource_key, ValidationResultIdentifier): run_id = resource_key.run_id run_name = run_id.run_name run_time = run_id.run_time expectation_suite_name = ( resource_key.expectation_suite_identifier. expectation_suite_name) if self.name == "profiling": logger.debug( " Rendering profiling for batch {}".format( resource_key.batch_identifier)) else: logger.debug( " Rendering validation: run name: {}, run time: {}, suite {} for batch {}" .format( run_name, run_time, expectation_suite_name, resource_key.batch_identifier, )) try: rendered_content = self.renderer_class.render(resource) viewable_content = self.view_class.render( rendered_content, data_context_id=self.data_context_id, show_how_to_buttons=self.show_how_to_buttons, ) self.target_store.set( SiteSectionIdentifier( site_section_name=self.name, resource_identifier=resource_key, ), viewable_content, ) except Exception as e: exception_message = f"""\ An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \ not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \ diagnose and repair the underlying issue. Detailed information follows: """ exception_traceback = traceback.format_exc() exception_message += (f'{type(e).__name__}: "{str(e)}". ' f'Traceback: "{exception_traceback}".') logger.error(exception_message, e, exc_info=True)