def _render_variable_types(cls, evrs, content_blocks) -> None: column_types = cls._get_column_types(evrs) # TODO: check if we have the information to make this statement. Do all columns have type expectations? column_type_counter = Counter(column_types.values()) table_rows = [ [type, str(column_type_counter[type])] for type in ["int", "float", "string", "datetime", "bool", "unknown"] ] content_blocks.append( RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Variable types", "tag": "h6", }, }), "table": table_rows, "styling": { "classes": ["col-6", "table-responsive", "mt-1", "p-1"], "body": { "classes": ["table", "table-sm"] }, }, }))
def _get_quantile_values_observed_value(cls, evr): if evr.result is None or evr.result.get("observed_value") is None: return "--" quantiles = evr.result.get("observed_value", {}).get("quantiles", []) value_ranges = evr.result.get("observed_value", {}).get("values", []) table_header_row = ["Quantile", "Value"] table_rows = [] quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"} for idx, quantile in enumerate(quantiles): quantile_string = quantile_strings.get(quantile) table_rows.append([ quantile_string if quantile_string else "{:3.2f}".format(quantile), str(value_ranges[idx]), ]) return RenderedTableContent( **{ "content_block_type": "table", "header_row": table_header_row, "table": table_rows, "styling": { "body": { "classes": ["table", "table-sm", "table-unbordered", "col-4"], } }, })
def render(cls, ge_object, header_row=None): """Each expectation method should return a list of rows""" if header_row is None: header_row = [] if isinstance(ge_object, list): table_entries = [] for sub_object in ge_object: expectation_type = cls._get_expectation_type(sub_object) extra_rows_fn = getattr(cls, expectation_type, None) if extra_rows_fn is not None: rows = extra_rows_fn(sub_object) table_entries.extend(rows) else: table_entries = [] expectation_type = cls._get_expectation_type(ge_object) extra_rows_fn = getattr(cls, expectation_type, None) if extra_rows_fn is not None: rows = extra_rows_fn(ge_object) table_entries.extend(rows) return RenderedTableContent( **{ "content_block_type": "table", "header_row": header_row, "table": table_entries, })
def render(cls, ge_object, header_row=None): """Each expectation method should return a list of rows""" if header_row is None: header_row = [] table_rows = [] if isinstance(ge_object, list): for sub_object in ge_object: expectation_type = cls._get_expectation_type(sub_object) if expectation_type in cls.expectation_renderers: new_rows = [ get_renderer_impl(expectation_type, renderer_type)[1](result=sub_object) for renderer_type in cls.expectation_renderers.get( expectation_type) ] table_rows.extend(new_rows) else: expectation_type = cls._get_expectation_type(ge_object) if expectation_type in cls.expectation_renderers: new_rows = [ get_renderer_impl(expectation_type, renderer_type)[1](result=ge_object) for renderer_type in cls.expectation_renderers.get( expectation_type) ] table_rows.extend(new_rows) return RenderedTableContent( **{ "content_block_type": "table", "header_row": header_row, "table": table_rows, })
def _get_table_content_block(cls, header="", subheader="", col=12): return RenderedTableContent( **{ "content_block_type": "table", "header": header, "subheader": subheader, "table": [ ["", "column_1", "column_2"], [ "row_1", cls._get_bullet_list_content_block( subheader="Nested Bullet List Content Block"), "buffalo", ], ["row_2", "crayon", "derby"], ], "styling": { "classes": [f"col-{col}", "table-responsive"], "styles": { "margin-top": "20px" }, "body": { "classes": ["table", "table-sm"] }, }, })
def test_render_table_component(): table_component_content = RenderedTableContent(**{ "content_block_type": "table", "header": "Overview", "table": [ ["Mean", "446"], ["Minimum", "1"], ], "styling": { "classes": ["col-4"], } }).to_json_dict() rendered_doc = ge.render.view.view.DefaultJinjaComponentView().render( { "content_block": table_component_content, "section_loop": {"index": 1}, "content_block_loop": {"index": 2}, } ) print(rendered_doc) rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "") assert rendered_doc == \ """ <div id="section-1-content-block-2" class="col-4" > <div id="section-1-content-block-2-header" > <h5> <span>Overview</span> </h5> </div> <table id="section-1-content-block-2-body" > <tr> <td id="section-1-content-block-2-cell-1-1" ><div class="show-scrollbars"><span>Mean</span></div></td><td id="section-1-content-block-2-cell-1-2" ><div class="show-scrollbars"><span>446</span></div></td></tr><tr> <td id="section-1-content-block-2-cell-2-1" ><div class="show-scrollbars"><span>Minimum</span></div></td><td id="section-1-content-block-2-cell-2-2" ><div class="show-scrollbars"><span>1</span></div></td></tr></table> </div>""".replace(" ", "").replace("\t", "").replace("\n", "")
def _render_dataset_info(cls, evrs, content_blocks) -> None: expect_table_row_count_to_be_between_evr = cls._find_evr_by_type( evrs["results"], "expect_table_row_count_to_be_between") table_rows = [] table_rows.append([ "Number of variables", len(cls._get_column_list_from_evrs(evrs)), ]) table_rows.append([ RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Number of observations", "tooltip": { "content": "expect_table_row_count_to_be_between" }, "params": { "tooltip_text": "Number of observations" }, }, }), "--" if not expect_table_row_count_to_be_between_evr else expect_table_row_count_to_be_between_evr.result["observed_value"], ]) table_rows += [ [ "Missing cells", cls._get_percentage_missing_cells_str(evrs), ], # ["Duplicate rows", "0 (0.0%)", ], #TODO: bring back when we have an expectation for this ] content_blocks.append( RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Dataset info", "tag": "h6", }, }), "table": table_rows, "styling": { "classes": ["col-6", "mt-1", "p-1"], "body": { "classes": ["table", "table-sm"] }, }, }))
def _get_unexpected_table(cls, evr): try: result = evr.result except KeyError: return None if result is None: return None if not result.get("partial_unexpected_list") and not result.get( "partial_unexpected_counts"): return None table_rows = [] if result.get("partial_unexpected_counts"): header_row = ["Unexpected Value", "Count"] for unexpected_count in result.get("partial_unexpected_counts"): if unexpected_count.get("value"): table_rows.append([ unexpected_count.get("value"), unexpected_count.get("count") ]) elif unexpected_count.get("value") == "": table_rows.append(["EMPTY", unexpected_count.get("count")]) elif unexpected_count.get("value") is not None: table_rows.append([ unexpected_count.get("value"), unexpected_count.get("count") ]) else: table_rows.append(["null", unexpected_count.get("count")]) else: header_row = ["Unexpected Value"] for unexpected_value in result.get("partial_unexpected_list"): if unexpected_value: table_rows.append([unexpected_value]) elif unexpected_value == "": table_rows.append(["EMPTY"]) elif unexpected_value is not None: table_rows.append([unexpected_value]) else: table_rows.append(["null"]) unexpected_table_content_block = RenderedTableContent( **{ "content_block_type": "table", "table": table_rows, "header_row": header_row, "styling": { "body": { "classes": ["table-bordered", "table-sm", "mt-3"] } }, }) return unexpected_table_content_block
def _render_quantile_table(cls, evrs): table_rows = [] quantile_evr = cls._find_evr_by_type( evrs, "expect_column_quantile_values_to_be_between") if not quantile_evr or quantile_evr.exception_info["raised_exception"]: return quantiles = quantile_evr.result["observed_value"]["quantiles"] quantile_ranges = quantile_evr.result["observed_value"]["values"] quantile_strings = {.25: "Q1", .75: "Q3", .50: "Median"} for idx, quantile in enumerate(quantiles): quantile_string = quantile_strings.get(quantile) table_rows.append([ { "content_block_type": "string_template", "string_template": { "template": quantile_string if quantile_string else "{:3.2f}".format(quantile), "tooltip": { "content": "expect_column_quantile_values_to_be_between \n expect_column_median_to_be_between" if quantile == 0.50 else "expect_column_quantile_values_to_be_between" } } }, quantile_ranges[idx], ]) return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": 'Quantiles', "tag": "h6" } }), "table": table_rows, "styling": { "classes": ["col-3", "mt-1", "pl-1", "pr-1"], "body": { "classes": ["table", "table-sm", "table-unbordered"], } }, })
def _descriptive_quantile_table_renderer( cls, configuration=None, result=None, language=None, runtime_configuration=None, **kwargs, ): assert result, "Must pass in result." table_rows = [] quantiles = result.result["observed_value"]["quantiles"] quantile_ranges = result.result["observed_value"]["values"] quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"} for idx, quantile in enumerate(quantiles): quantile_string = quantile_strings.get(quantile) table_rows.append([ { "content_block_type": "string_template", "string_template": { "template": quantile_string if quantile_string else f"{quantile:3.2f}", "tooltip": { "content": "expect_column_quantile_values_to_be_between \n expect_column_median_to_be_between" if quantile == 0.50 else "expect_column_quantile_values_to_be_between" }, }, }, quantile_ranges[idx], ]) return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Quantiles", "tag": "h6" }, }), "table": table_rows, "styling": { "classes": ["col-3", "mt-1", "pl-1", "pr-1"], "body": { "classes": ["table", "table-sm", "table-unbordered"], }, }, })
def _render_validation_info(cls, validation_results): run_id = validation_results.meta["run_id"] if isinstance(run_id, str): try: run_time = parse(run_id).strftime("%Y-%m-%dT%H:%M:%SZ") except (ValueError, TypeError): run_time = "__none__" run_name = run_id elif isinstance(run_id, dict): run_name = run_id.get("run_name") or "__none__" try: run_time = str( parse( run_id.get("run_time")).strftime("%Y-%m-%dT%H:%M:%SZ")) except (ValueError, TypeError): run_time = "__none__" elif isinstance(run_id, RunIdentifier): run_name = run_id.run_name or "__none__" run_time = run_id.run_time.strftime("%Y-%m-%dT%H:%M:%SZ") # TODO: Deprecate "great_expectations.__version__" ge_version = validation_results.meta.get( "great_expectations_version") or validation_results.meta.get( "great_expectations.__version__") return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Info", "tag": "h6", "styling": { "classes": ["m-0"] }, }, }), "table": [ ["Great Expectations Version", ge_version], ["Run Name", run_name], ["Run Time", run_time], ], "styling": { "classes": ["col-12", "table-responsive", "mt-1"], "body": { "classes": ["table", "table-sm"], "styles": { "margin-bottom": "0.5rem !important", "margin-top": "0.5rem !important", }, }, }, })
def test_render_section_page(): section = RenderedSectionContent( **{ "section_name": None, "content_blocks": [ RenderedHeaderContent( **{ "content_block_type": "header", "header": "Overview", } ), RenderedTableContent( **{ "content_block_type": "table", "header": "Dataset info", "table": [ ["Number of variables", "12"], ["Number of observations", "891"], ], "styling": { "classes": ["col-6", "table-responsive"], "styles": {"margin-top": "20px"}, "body": {"classes": ["table", "table-sm"]}, }, } ), ], } ) rendered_doc = ge.render.view.view.DefaultMarkdownPageView().render( RenderedDocumentContent(sections=[section]) ) rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "") assert ( rendered_doc == """ #ValidationResults ##Overview ###Datasetinfo ||||------------|------------|Numberofvariables|12Numberofobservations|891 ----------------------------------------------------------- Poweredby[GreatExpectations](https://greatexpectations.io/) """.replace( " ", "" ) .replace("\t", "") .replace("\n", "") )
def _render_validation_statistics(cls, validation_results): statistics = validation_results.statistics statistics_dict = OrderedDict([ ("evaluated_expectations", "Evaluated Expectations"), ("successful_expectations", "Successful Expectations"), ("unsuccessful_expectations", "Unsuccessful Expectations"), ("success_percent", "Success Percent"), ]) table_rows = [] for key, value in statistics_dict.items(): if statistics.get(key) is not None: if key == "success_percent": # table_rows.append([value, "{0:.2f}%".format(statistics[key])]) table_rows.append([ value, num_to_str(statistics[key], precision=4) + "%" ]) else: table_rows.append([value, statistics[key]]) return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Statistics", "tag": "h6", "styling": { "classes": ["m-0"] }, }, }), "table": table_rows, "styling": { "classes": ["col-6", "table-responsive", "mt-1", "p-1"], "body": { "classes": ["table", "table-sm"], "styles": { "margin-bottom": "0.5rem !important", "margin-top": "0.5rem !important", }, }, }, })
def _diagnostic_observed_value_renderer( cls, configuration=None, result=None, language=None, runtime_configuration=None, **kwargs, ): observed_value = result.result.get("observed_value") column_A = result.expectation_config.kwargs["column_A"] column_B = result.expectation_config.kwargs["column_B"] crosstab = result.result.get("details", {}).get("crosstab") if observed_value is not None: observed_value = num_to_str(observed_value, precision=3, use_locale=True) if crosstab is not None: table = [[""] + list(crosstab.columns)] for col in range(len(crosstab)): table.append([crosstab.index[col]] + list(crosstab.iloc[col, :])) return RenderedTableContent( **{ "content_block_type": "table", "header": f"Observed cramers phi of {observed_value}. \n" f"Crosstab between {column_A} (rows) and {column_B} (columns):", "table": table, "styling": { "body": { "classes": [ "table", "table-sm", "table-unbordered", "col-4", "mt-2", ], } }, }) else: return observed_value else: return "--"
def _render_stats_table(cls, evrs): expectation_renderers = { "expect_column_mean_to_be_between": "renderer.descriptive.stats_table.mean_row", "expect_column_min_to_be_between": "renderer.descriptive.stats_table.min_row", "expect_column_max_to_be_between": "renderer.descriptive.stats_table.max_row", } table_rows = [] for expectation_type, renderer_type in expectation_renderers.items(): evr = cls._find_evr_by_type(evrs, expectation_type) if evr and not evr.exception_info["raised_exception"]: renderer_impl = get_renderer_impl( object_name=expectation_type, renderer_type=renderer_type)[1] table_rows.append(renderer_impl(result=evr)) if len(table_rows) > 0: return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Statistics", "tag": "h6" }, }), "table": table_rows, "styling": { "classes": ["col-3", "mt-1", "pl-1", "pr-1"], "body": { "classes": ["table", "table-sm", "table-unbordered"], }, }, }) else: return
def _diagnostic_observed_value_renderer( cls, configuration=None, result=None, language=None, runtime_configuration=None, **kwargs, ): if result.result is None or result.result.get( "observed_value") is None: return "--" quantiles = result.result.get("observed_value", {}).get("quantiles", []) value_ranges = result.result.get("observed_value", {}).get("values", []) table_header_row = ["Quantile", "Value"] table_rows = [] quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"} for idx, quantile in enumerate(quantiles): quantile_string = quantile_strings.get(quantile) table_rows.append([ quantile_string if quantile_string else f"{quantile:3.2f}", str(value_ranges[idx]), ]) return RenderedTableContent( **{ "content_block_type": "table", "header_row": table_header_row, "table": table_rows, "styling": { "body": { "classes": ["table", "table-sm", "table-unbordered", "col-4"], } }, })
def _render_expectation_suite_info(cls, expectations): expectation_suite_name = expectations.expectation_suite_name # TODO: Deprecate "great_expectations.__version__" ge_version = expectations.meta.get( "great_expectations_version") or expectations.meta.get( "great_expectations.__version__") return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "Info", "tag": "h6", "styling": { "classes": ["m-0"] }, }, }), "table": [ ["Expectation Suite Name", expectation_suite_name], ["Great Expectations Version", ge_version], ], "styling": { "classes": ["col-12", "table-responsive", "mt-1"], "body": { "classes": ["table", "table-sm"], "styles": { "margin-bottom": "0.5rem !important", "margin-top": "0.5rem !important", }, }, }, })
def _render_nested_table_from_dict(cls, input_dict, header=None, sub_table=False): table_rows = [] for kwarg, value in input_dict.items(): if not isinstance(value, (dict, OrderedDict)): table_row = [ RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$value", "params": { "value": str(kwarg) }, "styling": { "default": { "styles": { "word-break": "break-all" } }, }, }, "styling": { "parent": { "classes": ["pr-3"], } }, }), RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$value", "params": { "value": str(value) }, "styling": { "default": { "styles": { "word-break": "break-all" } }, }, }, "styling": { "parent": { "classes": [], } }, }), ] else: table_row = [ RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$value", "params": { "value": str(kwarg) }, "styling": { "default": { "styles": { "word-break": "break-all" } }, }, }, "styling": { "parent": { "classes": ["pr-3"], } }, }), cls._render_nested_table_from_dict(value, sub_table=True), ] table_rows.append(table_row) table_rows.sort( key=lambda row: row[0].string_template["params"]["value"]) if sub_table: return RenderedTableContent( **{ "content_block_type": "table", "table": table_rows, "styling": { "classes": ["col-6", "table-responsive"], "body": { "classes": ["table", "table-sm", "m-0"] }, "parent": { "classes": ["pt-0", "pl-0", "border-top-0"] }, }, }) else: return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": header, "tag": "h6", "styling": { "classes": ["m-0"] }, }, }), "table": table_rows, "styling": { "body": { "classes": ["table", "table-sm"], "styles": { "margin-bottom": "0.5rem !important", "margin-top": "0.5rem !important", }, } }, })
def test_rendering_components_with_styling(): # Medium-complicated example to verify that all the things are correctly piped to all the places header_component_content = RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$var1 $var2 $var3", "params": { "var1": "AAA", "var2": "BBB", "var3": "CCC", }, "styling": { "default": { "classes": ["x"] }, "params": { "var1": { "classes": ["y"] } }, }, }, }), "subheader": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$var1 $var2 $var3", "params": { "var1": "aaa", "var2": "bbb", "var3": "ccc", }, "styling": { "default": { "classes": ["xx"] }, "params": { "var1": { "classes": ["yy"] } }, }, }, }), "table": [ ["Mean", "446"], ["Minimum", "1"], ], "styling": { "classes": ["root_foo"], "styles": { "root": "bar" }, "attributes": { "root": "baz" }, "header": { "classes": ["header_foo"], "styles": { "header": "bar" }, "attributes": { "header": "baz" }, }, "subheader": { "classes": ["subheader_foo"], "styles": { "subheader": "bar" }, "attributes": { "subheader": "baz" }, }, "body": { "classes": ["body_foo"], "styles": { "body": "bar" }, "attributes": { "body": "baz" }, }, }, }).to_json_dict() rendered_doc = ge.render.view.view.DefaultJinjaComponentView().render({ "content_block": header_component_content, "section_loop": { "index": 1 }, "content_block_loop": { "index": 2 }, }) print(rendered_doc) rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "") assert (rendered_doc == """ <div id="section-1-content-block-2" class="root_foo" root="baz" style="root:bar;" > <div id="section-1-content-block-2-header" class="header_foo" header="baz" style="header:bar;" > <div> <span > <span class="y" >AAA</span> <span class="x" >BBB</span> <span class="x" >CCC</span> </span> </div> <div id="section-1-content-block-2-subheader" class="subheader_foo" subheader="baz" style="subheader:bar;" > <span > <span class="yy" >aaa</span> <span class="xx" >bbb</span> <span class="xx" >ccc</span> </span> </div> </div> <table id="section-1-content-block-2-body" class="body_foo" body="baz" style="body:bar;" data-toggle="table" > <thead hidden> <tr> <th> </th> <th> </th> </tr> </thead> <tbody> <tr> <td id="section-1-content-block-2-cell-1-1" ><div class="show-scrollbars">Mean</div></td><td id="section-1-content-block-2-cell-1-2" ><div class="show-scrollbars">446</div></td></tr><tr> <td id="section-1-content-block-2-cell-2-1" ><div class="show-scrollbars">Minimum</div></td><td id="section-1-content-block-2-cell-2-2" ><div class="show-scrollbars">1</div></td></tr></tbody> </table> </div>""".replace(" ", "").replace("\t", "").replace("\n", ""))
def render(cls, index_links_dict): sections = [] cta_object = index_links_dict.pop("cta_object", None) try: content_blocks = [] # site name header site_name_header_block = RenderedHeaderContent( **{ "content_block_type": "header", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$title_prefix | $site_name", "params": { "site_name": index_links_dict.get( "site_name"), "title_prefix": "Data Docs" }, "styling": { "params": { "title_prefix": { "tag": "strong" } } }, } }), "styling": { "classes": ["col-12", "ge-index-page-site-name-title"], "header": { "classes": ["alert", "alert-secondary"] } } }) content_blocks.append(site_name_header_block) table_rows = [] table_header_row = [] link_list_keys_to_render = [] header_dict = OrderedDict( [["expectations_links", "Expectation Suite"], ["validations_links", "Validation Results (run_id)"]]) for link_lists_key, header in header_dict.items(): if index_links_dict.get(link_lists_key): class_header_str = link_lists_key.replace("_", "-") class_str = "ge-index-page-table-{}-header".format( class_header_str) header = RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": header, "params": {}, "styling": { "classes": [class_str], } } }) table_header_row.append(header) link_list_keys_to_render.append(link_lists_key) generator_table = RenderedTableContent( **{ "content_block_type": "table", "header_row": table_header_row, "table": table_rows, "styling": { "classes": ["col-12", "ge-index-page-table-container"], "styles": { "margin-top": "10px" }, "body": { "classes": [ "table", "table-sm", "ge-index-page-generator-table" ] } } }) table_rows += cls._generate_links_table_rows( index_links_dict, link_list_keys_to_render=link_list_keys_to_render) content_blocks.append(generator_table) if index_links_dict.get("profiling_links"): profiling_table_rows = [] for profiling_link_dict in index_links_dict.get( "profiling_links"): profiling_table_rows.append([ RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$link_text", "params": { "link_text": profiling_link_dict[ "expectation_suite_name"] + "." + profiling_link_dict["batch_identifier"] }, "tag": "a", "styling": { "attributes": { "href": profiling_link_dict["filepath"] }, "classes": [ "ge-index-page-table-expectation-suite-link" ] } }, }) ]) content_blocks.append( RenderedTableContent( **{ "content_block_type": "table", "header_row": ["Profiling Results"], "table": profiling_table_rows, "styling": { "classes": ["col-12", "ge-index-page-table-container"], "styles": { "margin-top": "10px" }, "body": { "classes": [ "table", "table-sm", "ge-index-page-generator-table" ] } } })) section = RenderedSectionContent( **{ "section_name": index_links_dict.get("site_name"), "content_blocks": content_blocks }) sections.append(section) index_page_document = RenderedDocumentContent( **{ "renderer_type": "SiteIndexPageRenderer", "utm_medium": "index-page", "sections": sections }) if cta_object: index_page_document.cta_footer = CallToActionRenderer.render( cta_object) return index_page_document except Exception as e: exception_message = f'''\ An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \ not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \ diagnose and repair the underlying issue. Detailed information follows: ''' exception_traceback = traceback.format_exc() exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' logger.error(exception_message, e, exc_info=True)
def render(cls, index_links_dict): sections = [] cta_object = index_links_dict.pop("cta_object", None) try: content_blocks = [] # site name header site_name_header_block = RenderedHeaderContent( **{ "content_block_type": "header", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$title_prefix | $site_name", "params": { "site_name": index_links_dict.get( "site_name"), "title_prefix": "Data Docs" }, "styling": { "params": { "title_prefix": { "tag": "strong" } } }, } }), "styling": { "classes": ["col-12", "ge-index-page-site-name-title"], "header": { "classes": ["alert", "alert-secondary"] } } }) content_blocks.append(site_name_header_block) table_rows = [] table_header_row = [] link_list_keys_to_render = [] header_dict = OrderedDict( [["expectations_links", "Expectation Suite"], ["validations_links", "Validation Results (run_id)"]]) for link_lists_key, header in header_dict.items(): if index_links_dict.get(link_lists_key): class_header_str = link_lists_key.replace("_", "-") class_str = "ge-index-page-table-{}-header".format( class_header_str) header = RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": header, "params": {}, "styling": { "classes": [class_str], } } }) table_header_row.append(header) link_list_keys_to_render.append(link_lists_key) generator_table = RenderedTableContent( **{ "content_block_type": "table", "header_row": table_header_row, "table": table_rows, "styling": { "classes": ["col-12", "ge-index-page-table-container"], "styles": { "margin-top": "10px" }, "body": { "classes": [ "table", "table-sm", "ge-index-page-generator-table" ] } } }) table_rows += cls._generate_links_table_rows( index_links_dict, link_list_keys_to_render=link_list_keys_to_render) content_blocks.append(generator_table) if index_links_dict.get("profiling_links"): profiling_table_rows = [] for profiling_link_dict in index_links_dict.get( "profiling_links"): profiling_table_rows.append([ RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": "$link_text", "params": { "link_text": profiling_link_dict[ "expectation_suite_name"] + "." + profiling_link_dict["batch_identifier"] }, "tag": "a", "styling": { "attributes": { "href": profiling_link_dict["filepath"] }, "classes": [ "ge-index-page-table-expectation-suite-link" ] } }, }) ]) content_blocks.append( RenderedTableContent( **{ "content_block_type": "table", "header_row": ["Profiling Results"], "table": profiling_table_rows, "styling": { "classes": ["col-12", "ge-index-page-table-container"], "styles": { "margin-top": "10px" }, "body": { "classes": [ "table", "table-sm", "ge-index-page-generator-table" ] } } })) section = RenderedSectionContent( **{ "section_name": index_links_dict.get("site_name"), "content_blocks": content_blocks }) sections.append(section) index_page_document = RenderedDocumentContent( **{ "renderer_type": "SiteIndexPageRenderer", "utm_medium": "index-page", "sections": sections }) if cta_object: index_page_document.cta_footer = CallToActionRenderer.render( cta_object) return index_page_document except Exception as e: logger.error("Exception occurred during data docs rendering: ", e, exc_info=True)
def _render_stats_table(cls, evrs): table_rows = [] mean_evr = cls._find_evr_by_type(evrs, "expect_column_mean_to_be_between") if not mean_evr or mean_evr.exception_info["raised_exception"]: return mean_value = "{:.2f}".format( mean_evr.result['observed_value']) if mean_evr else None if mean_value: table_rows.append([{ "content_block_type": "string_template", "string_template": { "template": "Mean", "tooltip": { "content": "expect_column_mean_to_be_between" } } }, mean_value]) min_evr = cls._find_evr_by_type(evrs, "expect_column_min_to_be_between") min_value = "{:.2f}".format( min_evr.result['observed_value']) if min_evr else None if min_value: table_rows.append([ { "content_block_type": "string_template", "string_template": { "template": "Minimum", "tooltip": { "content": "expect_column_min_to_be_between" } } }, min_value, ]) max_evr = cls._find_evr_by_type(evrs, "expect_column_max_to_be_between") max_value = "{:.2f}".format( max_evr.result['observed_value']) if max_evr else None if max_value: table_rows.append([{ "content_block_type": "string_template", "string_template": { "template": "Maximum", "tooltip": { "content": "expect_column_max_to_be_between" } } }, max_value]) if len(table_rows) > 0: return RenderedTableContent( **{ "content_block_type": "table", "header": RenderedStringTemplateContent( **{ "content_block_type": "string_template", "string_template": { "template": 'Statistics', "tag": "h6" } }), "table": table_rows, "styling": { "classes": ["col-3", "mt-1", "pl-1", "pr-1"], "body": { "classes": ["table", "table-sm", "table-unbordered"], } }, }) else: return
def _get_unexpected_table(cls, evr): try: result = evr.result except KeyError: return None if result is None: return None if not result.get("partial_unexpected_list") and not result.get( "partial_unexpected_counts" ): return None table_rows = [] if result.get("partial_unexpected_counts"): # We will check to see whether we have *all* of the unexpected values # accounted for in our count, and include counts if we do. If we do not, # we will use this as simply a better (non-repeating) source of # "sampled" unexpected values total_count = 0 for unexpected_count_dict in result.get("partial_unexpected_counts"): if not isinstance(unexpected_count_dict, dict): # handles case: "partial_exception_counts requires a hashable type" # this case is also now deprecated (because the error is moved to an errors key # the error also *should have* been updated to "partial_unexpected_counts ..." long ago. # NOTE: JPC 20200724 - Consequently, this codepath should be removed by approximately Q1 2021 continue value = unexpected_count_dict.get("value") count = unexpected_count_dict.get("count") total_count += count if value is not None and value != "": table_rows.append([value, count]) elif value == "": table_rows.append(["EMPTY", count]) else: table_rows.append(["null", count]) # Check to see if we have *all* of the unexpected values accounted for. If so, # we show counts. If not, we only show "sampled" unexpected values. if total_count == result.get("unexpected_count"): header_row = ["Unexpected Value", "Count"] else: header_row = ["Sampled Unexpected Values"] table_rows = [[row[0]] for row in table_rows] else: header_row = ["Sampled Unexpected Values"] sampled_values_set = set() for unexpected_value in result.get("partial_unexpected_list"): if unexpected_value: string_unexpected_value = unexpected_value elif unexpected_value == "": string_unexpected_value = "EMPTY" else: string_unexpected_value = "null" if string_unexpected_value not in sampled_values_set: table_rows.append([string_unexpected_value]) sampled_values_set.add(string_unexpected_value) unexpected_table_content_block = RenderedTableContent( **{ "content_block_type": "table", "table": table_rows, "header_row": header_row, "styling": { "body": {"classes": ["table-bordered", "table-sm", "mt-3"]} }, } ) return unexpected_table_content_block
def _get_unexpected_table(cls, evr): try: result = evr.result except KeyError: return None if result is None: return None if not result.get("partial_unexpected_list") and not result.get( "partial_unexpected_counts"): return None table_rows = [] if result.get("partial_unexpected_counts"): header_row = ["Unexpected Value", "Count"] for unexpected_count in result.get("partial_unexpected_counts"): if not isinstance(unexpected_count, dict): # handles case: "partial_exception_counts requires a hashable type" # this case is also now deprecated (because the error is moved to an errors key # the error also *should have* been updated to "partial_unexpected_counts ..." long ago. # NOTE: JPC 20200724 - Consequently, this codepath should be removed by approximately Q1 2021 continue elif unexpected_count.get("value"): table_rows.append([ unexpected_count.get("value"), unexpected_count.get("count") ]) elif unexpected_count.get("value") == "": table_rows.append(["EMPTY", unexpected_count.get("count")]) elif unexpected_count.get("value") is not None: table_rows.append([ unexpected_count.get("value"), unexpected_count.get("count") ]) else: table_rows.append(["null", unexpected_count.get("count")]) else: header_row = ["Unexpected Value"] for unexpected_value in result.get("partial_unexpected_list"): if unexpected_value: table_rows.append([unexpected_value]) elif unexpected_value == "": table_rows.append(["EMPTY"]) elif unexpected_value is not None: table_rows.append([unexpected_value]) else: table_rows.append(["null"]) unexpected_table_content_block = RenderedTableContent( **{ "content_block_type": "table", "table": table_rows, "header_row": header_row, "styling": { "body": { "classes": ["table-bordered", "table-sm", "mt-3"] } }, }) return unexpected_table_content_block
def test_render_section_page(): section = RenderedSectionContent( **{ "section_name": None, "content_blocks": [ RenderedHeaderContent(**{ "content_block_type": "header", "header": "Overview", }), RenderedTableContent( **{ "content_block_type": "table", "header": "Dataset info", "table": [ ["Number of variables", "12"], ["Number of observations", "891"], ], "styling": { "classes": ["col-6", "table-responsive"], "styles": { "margin-top": "20px" }, "body": { "classes": ["table", "table-sm"] }, }, }), ], }).to_json_dict() rendered_doc = ge.render.view.view.DefaultJinjaSectionView().render({ "section": section, "section_loop": { "index": 1 }, }) # .replace(" ", "").replace("\t", "").replace("\n", "") print(rendered_doc) rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "") assert ( rendered_doc == """<div id="section-1" class="ge-section container-fluid mb-1 pb-1 pl-sm-3 px-0"> <div class="row" > <div id="content-block-1" > <div id="content-block-1-header" > <h5> Overview </h5> </div> </div> <div id="content-block-2" class="col-6 table-responsive" style="margin-top:20px;" > <div id="content-block-2-header" > <h5> Dataset info </h5> </div> <table id="content-block-2-body" class="table table-sm" data-toggle="table" > <thead hidden> <tr> <th> </th> <th> </th> </tr> </thead> <tbody> <tr> <td id="content-block-2-cell-1-1" ><div class="show-scrollbars">Number of variables</div></td><td id="content-block-2-cell-1-2" ><div class="show-scrollbars">12</div></td></tr><tr> <td id="content-block-2-cell-2-1" ><div class="show-scrollbars">Number of observations</div></td><td id="content-block-2-cell-2-2" ><div class="show-scrollbars">891</div></td></tr></tbody> </table> </div> </div> </div>""".replace(" ", "").replace("\t", "").replace("\n", ""))
def _prescriptive_renderer(cls, configuration=None, result=None, language=None, runtime_configuration=None, **kwargs): runtime_configuration = runtime_configuration or {} include_column_name = runtime_configuration.get( "include_column_name", True) include_column_name = (include_column_name if include_column_name is not None else True) styling = runtime_configuration.get("styling") params = substitute_none_for_missing( configuration["kwargs"], ["column", "quantile_ranges", "row_condition", "condition_parser"], ) template_str = "quantiles must be within the following value ranges." if include_column_name: template_str = "$column " + template_str if params["row_condition"] is not None: ( conditional_template_str, conditional_params, ) = parse_row_condition_string_pandas_engine( params["row_condition"]) template_str = (conditional_template_str + ", then " + template_str[0].lower() + template_str[1:]) params.update(conditional_params) expectation_string_obj = { "content_block_type": "string_template", "string_template": { "template": template_str, "params": params }, } quantiles = params["quantile_ranges"]["quantiles"] value_ranges = params["quantile_ranges"]["value_ranges"] table_header_row = ["Quantile", "Min Value", "Max Value"] table_rows = [] quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"} for quantile, value_range in zip(quantiles, value_ranges): quantile_string = quantile_strings.get(quantile, "{:3.2f}".format(quantile)) table_rows.append([ quantile_string, str(value_range[0]) if value_range[0] is not None else "Any", str(value_range[1]) if value_range[1] is not None else "Any", ]) quantile_range_table = RenderedTableContent( **{ "content_block_type": "table", "header_row": table_header_row, "table": table_rows, "styling": { "body": { "classes": [ "table", "table-sm", "table-unbordered", "col-4", "mt-2", ], }, "parent": { "styles": { "list-style-type": "none" } }, }, }) return [expectation_string_obj, quantile_range_table]