def _render_variable_types(cls, evrs, content_blocks) -> None:
column_types = cls._get_column_types(evrs)
# TODO: check if we have the information to make this statement. Do all columns have type expectations?
column_type_counter = Counter(column_types.values())
table_rows = [
[type, str(column_type_counter[type])] for type in
["int", "float", "string", "datetime", "bool", "unknown"]
]
content_blocks.append(
RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Variable types",
"tag": "h6",
},
}),
"table":
table_rows,
"styling": {
"classes":
["col-6", "table-responsive", "mt-1", "p-1"],
"body": {
"classes": ["table", "table-sm"]
},
},
}))
def _get_quantile_values_observed_value(cls, evr):
if evr.result is None or evr.result.get("observed_value") is None:
return "--"
quantiles = evr.result.get("observed_value", {}).get("quantiles", [])
value_ranges = evr.result.get("observed_value", {}).get("values", [])
table_header_row = ["Quantile", "Value"]
table_rows = []
quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"}
for idx, quantile in enumerate(quantiles):
quantile_string = quantile_strings.get(quantile)
table_rows.append([
quantile_string
if quantile_string else "{:3.2f}".format(quantile),
str(value_ranges[idx]),
])
return RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"body": {
"classes":
["table", "table-sm", "table-unbordered", "col-4"],
}
},
})
def render(cls, ge_object, header_row=None):
"""Each expectation method should return a list of rows"""
if header_row is None:
header_row = []
if isinstance(ge_object, list):
table_entries = []
for sub_object in ge_object:
expectation_type = cls._get_expectation_type(sub_object)
extra_rows_fn = getattr(cls, expectation_type, None)
if extra_rows_fn is not None:
rows = extra_rows_fn(sub_object)
table_entries.extend(rows)
else:
table_entries = []
expectation_type = cls._get_expectation_type(ge_object)
extra_rows_fn = getattr(cls, expectation_type, None)
if extra_rows_fn is not None:
rows = extra_rows_fn(ge_object)
table_entries.extend(rows)
return RenderedTableContent(
**{
"content_block_type": "table",
"header_row": header_row,
"table": table_entries,
})
def render(cls, ge_object, header_row=None):
"""Each expectation method should return a list of rows"""
if header_row is None:
header_row = []
table_rows = []
if isinstance(ge_object, list):
for sub_object in ge_object:
expectation_type = cls._get_expectation_type(sub_object)
if expectation_type in cls.expectation_renderers:
new_rows = [
get_renderer_impl(expectation_type,
renderer_type)[1](result=sub_object)
for renderer_type in cls.expectation_renderers.get(
expectation_type)
]
table_rows.extend(new_rows)
else:
expectation_type = cls._get_expectation_type(ge_object)
if expectation_type in cls.expectation_renderers:
new_rows = [
get_renderer_impl(expectation_type,
renderer_type)[1](result=ge_object)
for renderer_type in cls.expectation_renderers.get(
expectation_type)
]
table_rows.extend(new_rows)
return RenderedTableContent(
**{
"content_block_type": "table",
"header_row": header_row,
"table": table_rows,
})
def _get_table_content_block(cls, header="", subheader="", col=12):
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
header,
"subheader":
subheader,
"table": [
["", "column_1", "column_2"],
[
"row_1",
cls._get_bullet_list_content_block(
subheader="Nested Bullet List Content Block"),
"buffalo",
],
["row_2", "crayon", "derby"],
],
"styling": {
"classes": [f"col-{col}", "table-responsive"],
"styles": {
"margin-top": "20px"
},
"body": {
"classes": ["table", "table-sm"]
},
},
})
def test_render_table_component():
table_component_content = RenderedTableContent(**{
"content_block_type": "table",
"header": "Overview",
"table": [
["Mean", "446"],
["Minimum", "1"],
],
"styling": {
"classes": ["col-4"],
}
}).to_json_dict()
rendered_doc = ge.render.view.view.DefaultJinjaComponentView().render(
{
"content_block": table_component_content,
"section_loop": {"index": 1},
"content_block_loop": {"index": 2},
}
)
print(rendered_doc)
rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "")
assert rendered_doc == \
"""
<div id="section-1-content-block-2" class="col-4" >
<div id="section-1-content-block-2-header" >
<h5>
<span>Overview</span>
</h5>
</div>
<table id="section-1-content-block-2-body" >
<tr>
<td id="section-1-content-block-2-cell-1-1" ><div class="show-scrollbars"><span>Mean</span></div></td><td id="section-1-content-block-2-cell-1-2" ><div class="show-scrollbars"><span>446</span></div></td></tr><tr>
<td id="section-1-content-block-2-cell-2-1" ><div class="show-scrollbars"><span>Minimum</span></div></td><td id="section-1-content-block-2-cell-2-2" ><div class="show-scrollbars"><span>1</span></div></td></tr></table>
</div>""".replace(" ", "").replace("\t", "").replace("\n", "")
def _render_dataset_info(cls, evrs, content_blocks) -> None:
expect_table_row_count_to_be_between_evr = cls._find_evr_by_type(
evrs["results"], "expect_table_row_count_to_be_between")
table_rows = []
table_rows.append([
"Number of variables",
len(cls._get_column_list_from_evrs(evrs)),
])
table_rows.append([
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Number of observations",
"tooltip": {
"content": "expect_table_row_count_to_be_between"
},
"params": {
"tooltip_text": "Number of observations"
},
},
}),
"--" if not expect_table_row_count_to_be_between_evr else
expect_table_row_count_to_be_between_evr.result["observed_value"],
])
table_rows += [
[
"Missing cells",
cls._get_percentage_missing_cells_str(evrs),
],
# ["Duplicate rows", "0 (0.0%)", ], #TODO: bring back when we have an expectation for this
]
content_blocks.append(
RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Dataset info",
"tag": "h6",
},
}),
"table":
table_rows,
"styling": {
"classes": ["col-6", "mt-1", "p-1"],
"body": {
"classes": ["table", "table-sm"]
},
},
}))
def _get_unexpected_table(cls, evr):
try:
result = evr.result
except KeyError:
return None
if result is None:
return None
if not result.get("partial_unexpected_list") and not result.get(
"partial_unexpected_counts"):
return None
table_rows = []
if result.get("partial_unexpected_counts"):
header_row = ["Unexpected Value", "Count"]
for unexpected_count in result.get("partial_unexpected_counts"):
if unexpected_count.get("value"):
table_rows.append([
unexpected_count.get("value"),
unexpected_count.get("count")
])
elif unexpected_count.get("value") == "":
table_rows.append(["EMPTY", unexpected_count.get("count")])
elif unexpected_count.get("value") is not None:
table_rows.append([
unexpected_count.get("value"),
unexpected_count.get("count")
])
else:
table_rows.append(["null", unexpected_count.get("count")])
else:
header_row = ["Unexpected Value"]
for unexpected_value in result.get("partial_unexpected_list"):
if unexpected_value:
table_rows.append([unexpected_value])
elif unexpected_value == "":
table_rows.append(["EMPTY"])
elif unexpected_value is not None:
table_rows.append([unexpected_value])
else:
table_rows.append(["null"])
unexpected_table_content_block = RenderedTableContent(
**{
"content_block_type": "table",
"table": table_rows,
"header_row": header_row,
"styling": {
"body": {
"classes": ["table-bordered", "table-sm", "mt-3"]
}
},
})
return unexpected_table_content_block
def _render_quantile_table(cls, evrs):
table_rows = []
quantile_evr = cls._find_evr_by_type(
evrs, "expect_column_quantile_values_to_be_between")
if not quantile_evr or quantile_evr.exception_info["raised_exception"]:
return
quantiles = quantile_evr.result["observed_value"]["quantiles"]
quantile_ranges = quantile_evr.result["observed_value"]["values"]
quantile_strings = {.25: "Q1", .75: "Q3", .50: "Median"}
for idx, quantile in enumerate(quantiles):
quantile_string = quantile_strings.get(quantile)
table_rows.append([
{
"content_block_type": "string_template",
"string_template": {
"template":
quantile_string
if quantile_string else "{:3.2f}".format(quantile),
"tooltip": {
"content":
"expect_column_quantile_values_to_be_between \n expect_column_median_to_be_between"
if quantile == 0.50 else
"expect_column_quantile_values_to_be_between"
}
}
},
quantile_ranges[idx],
])
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": 'Quantiles',
"tag": "h6"
}
}),
"table":
table_rows,
"styling": {
"classes": ["col-3", "mt-1", "pl-1", "pr-1"],
"body": {
"classes": ["table", "table-sm", "table-unbordered"],
}
},
})
def _descriptive_quantile_table_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
assert result, "Must pass in result."
table_rows = []
quantiles = result.result["observed_value"]["quantiles"]
quantile_ranges = result.result["observed_value"]["values"]
quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"}
for idx, quantile in enumerate(quantiles):
quantile_string = quantile_strings.get(quantile)
table_rows.append([
{
"content_block_type": "string_template",
"string_template": {
"template": quantile_string
if quantile_string else f"{quantile:3.2f}",
"tooltip": {
"content":
"expect_column_quantile_values_to_be_between \n expect_column_median_to_be_between"
if quantile == 0.50 else
"expect_column_quantile_values_to_be_between"
},
},
},
quantile_ranges[idx],
])
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Quantiles",
"tag": "h6"
},
}),
"table":
table_rows,
"styling": {
"classes": ["col-3", "mt-1", "pl-1", "pr-1"],
"body": {
"classes": ["table", "table-sm", "table-unbordered"],
},
},
})
def _render_validation_info(cls, validation_results):
run_id = validation_results.meta["run_id"]
if isinstance(run_id, str):
try:
run_time = parse(run_id).strftime("%Y-%m-%dT%H:%M:%SZ")
except (ValueError, TypeError):
run_time = "__none__"
run_name = run_id
elif isinstance(run_id, dict):
run_name = run_id.get("run_name") or "__none__"
try:
run_time = str(
parse(
run_id.get("run_time")).strftime("%Y-%m-%dT%H:%M:%SZ"))
except (ValueError, TypeError):
run_time = "__none__"
elif isinstance(run_id, RunIdentifier):
run_name = run_id.run_name or "__none__"
run_time = run_id.run_time.strftime("%Y-%m-%dT%H:%M:%SZ")
# TODO: Deprecate "great_expectations.__version__"
ge_version = validation_results.meta.get(
"great_expectations_version") or validation_results.meta.get(
"great_expectations.__version__")
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Info",
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"table": [
["Great Expectations Version", ge_version],
["Run Name", run_name],
["Run Time", run_time],
],
"styling": {
"classes": ["col-12", "table-responsive", "mt-1"],
"body": {
"classes": ["table", "table-sm"],
"styles": {
"margin-bottom": "0.5rem !important",
"margin-top": "0.5rem !important",
},
},
},
})
def test_render_section_page():
section = RenderedSectionContent(
**{
"section_name": None,
"content_blocks": [
RenderedHeaderContent(
**{
"content_block_type": "header",
"header": "Overview",
}
),
RenderedTableContent(
**{
"content_block_type": "table",
"header": "Dataset info",
"table": [
["Number of variables", "12"],
["Number of observations", "891"],
],
"styling": {
"classes": ["col-6", "table-responsive"],
"styles": {"margin-top": "20px"},
"body": {"classes": ["table", "table-sm"]},
},
}
),
],
}
)
rendered_doc = ge.render.view.view.DefaultMarkdownPageView().render(
RenderedDocumentContent(sections=[section])
)
rendered_doc = rendered_doc.replace(" ", "").replace("\t", "").replace("\n", "")
assert (
rendered_doc
== """
#ValidationResults
##Overview
###Datasetinfo
||||------------|------------|Numberofvariables|12Numberofobservations|891
-----------------------------------------------------------
Poweredby[GreatExpectations](https://greatexpectations.io/)
""".replace(
" ", ""
)
.replace("\t", "")
.replace("\n", "")
)
def _render_validation_statistics(cls, validation_results):
statistics = validation_results.statistics
statistics_dict = OrderedDict([
("evaluated_expectations", "Evaluated Expectations"),
("successful_expectations", "Successful Expectations"),
("unsuccessful_expectations", "Unsuccessful Expectations"),
("success_percent", "Success Percent"),
])
table_rows = []
for key, value in statistics_dict.items():
if statistics.get(key) is not None:
if key == "success_percent":
# table_rows.append([value, "{0:.2f}%".format(statistics[key])])
table_rows.append([
value,
num_to_str(statistics[key], precision=4) + "%"
])
else:
table_rows.append([value, statistics[key]])
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Statistics",
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"table":
table_rows,
"styling": {
"classes": ["col-6", "table-responsive", "mt-1", "p-1"],
"body": {
"classes": ["table", "table-sm"],
"styles": {
"margin-bottom": "0.5rem !important",
"margin-top": "0.5rem !important",
},
},
},
})
def _diagnostic_observed_value_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
observed_value = result.result.get("observed_value")
column_A = result.expectation_config.kwargs["column_A"]
column_B = result.expectation_config.kwargs["column_B"]
crosstab = result.result.get("details", {}).get("crosstab")
if observed_value is not None:
observed_value = num_to_str(observed_value,
precision=3,
use_locale=True)
if crosstab is not None:
table = [[""] + list(crosstab.columns)]
for col in range(len(crosstab)):
table.append([crosstab.index[col]] +
list(crosstab.iloc[col, :]))
return RenderedTableContent(
**{
"content_block_type": "table",
"header":
f"Observed cramers phi of {observed_value}. \n"
f"Crosstab between {column_A} (rows) and {column_B} (columns):",
"table": table,
"styling": {
"body": {
"classes": [
"table",
"table-sm",
"table-unbordered",
"col-4",
"mt-2",
],
}
},
})
else:
return observed_value
else:
return "--"
def _render_stats_table(cls, evrs):
expectation_renderers = {
"expect_column_mean_to_be_between":
"renderer.descriptive.stats_table.mean_row",
"expect_column_min_to_be_between":
"renderer.descriptive.stats_table.min_row",
"expect_column_max_to_be_between":
"renderer.descriptive.stats_table.max_row",
}
table_rows = []
for expectation_type, renderer_type in expectation_renderers.items():
evr = cls._find_evr_by_type(evrs, expectation_type)
if evr and not evr.exception_info["raised_exception"]:
renderer_impl = get_renderer_impl(
object_name=expectation_type,
renderer_type=renderer_type)[1]
table_rows.append(renderer_impl(result=evr))
if len(table_rows) > 0:
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Statistics",
"tag": "h6"
},
}),
"table":
table_rows,
"styling": {
"classes": ["col-3", "mt-1", "pl-1", "pr-1"],
"body": {
"classes":
["table", "table-sm", "table-unbordered"],
},
},
})
else:
return
def _diagnostic_observed_value_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
if result.result is None or result.result.get(
"observed_value") is None:
return "--"
quantiles = result.result.get("observed_value",
{}).get("quantiles", [])
value_ranges = result.result.get("observed_value",
{}).get("values", [])
table_header_row = ["Quantile", "Value"]
table_rows = []
quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"}
for idx, quantile in enumerate(quantiles):
quantile_string = quantile_strings.get(quantile)
table_rows.append([
quantile_string if quantile_string else f"{quantile:3.2f}",
str(value_ranges[idx]),
])
return RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"body": {
"classes":
["table", "table-sm", "table-unbordered", "col-4"],
}
},
})
def _render_expectation_suite_info(cls, expectations):
expectation_suite_name = expectations.expectation_suite_name
# TODO: Deprecate "great_expectations.__version__"
ge_version = expectations.meta.get(
"great_expectations_version") or expectations.meta.get(
"great_expectations.__version__")
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Info",
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"table": [
["Expectation Suite Name", expectation_suite_name],
["Great Expectations Version", ge_version],
],
"styling": {
"classes": ["col-12", "table-responsive", "mt-1"],
"body": {
"classes": ["table", "table-sm"],
"styles": {
"margin-bottom": "0.5rem !important",
"margin-top": "0.5rem !important",
},
},
},
})
def _render_nested_table_from_dict(cls,
input_dict,
header=None,
sub_table=False):
table_rows = []
for kwarg, value in input_dict.items():
if not isinstance(value, (dict, OrderedDict)):
table_row = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(kwarg)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": ["pr-3"],
}
},
}),
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(value)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": [],
}
},
}),
]
else:
table_row = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(kwarg)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": ["pr-3"],
}
},
}),
cls._render_nested_table_from_dict(value, sub_table=True),
]
table_rows.append(table_row)
table_rows.sort(
key=lambda row: row[0].string_template["params"]["value"])
if sub_table:
return RenderedTableContent(
**{
"content_block_type": "table",
"table": table_rows,
"styling": {
"classes": ["col-6", "table-responsive"],
"body": {
"classes": ["table", "table-sm", "m-0"]
},
"parent": {
"classes": ["pt-0", "pl-0", "border-top-0"]
},
},
})
else:
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": header,
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"table":
table_rows,
"styling": {
"body": {
"classes": ["table", "table-sm"],
"styles": {
"margin-bottom": "0.5rem !important",
"margin-top": "0.5rem !important",
},
}
},
})
def test_rendering_components_with_styling():
# Medium-complicated example to verify that all the things are correctly piped to all the places
header_component_content = RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$var1 $var2 $var3",
"params": {
"var1": "AAA",
"var2": "BBB",
"var3": "CCC",
},
"styling": {
"default": {
"classes": ["x"]
},
"params": {
"var1": {
"classes": ["y"]
}
},
},
},
}),
"subheader":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$var1 $var2 $var3",
"params": {
"var1": "aaa",
"var2": "bbb",
"var3": "ccc",
},
"styling": {
"default": {
"classes": ["xx"]
},
"params": {
"var1": {
"classes": ["yy"]
}
},
},
},
}),
"table": [
["Mean", "446"],
["Minimum", "1"],
],
"styling": {
"classes": ["root_foo"],
"styles": {
"root": "bar"
},
"attributes": {
"root": "baz"
},
"header": {
"classes": ["header_foo"],
"styles": {
"header": "bar"
},
"attributes": {
"header": "baz"
},
},
"subheader": {
"classes": ["subheader_foo"],
"styles": {
"subheader": "bar"
},
"attributes": {
"subheader": "baz"
},
},
"body": {
"classes": ["body_foo"],
"styles": {
"body": "bar"
},
"attributes": {
"body": "baz"
},
},
},
}).to_json_dict()
rendered_doc = ge.render.view.view.DefaultJinjaComponentView().render({
"content_block":
header_component_content,
"section_loop": {
"index": 1
},
"content_block_loop": {
"index": 2
},
})
print(rendered_doc)
rendered_doc = rendered_doc.replace(" ", "").replace("\t",
"").replace("\n", "")
assert (rendered_doc == """
<div id="section-1-content-block-2" class="root_foo" root="baz" style="root:bar;" >
<div id="section-1-content-block-2-header" class="header_foo" header="baz" style="header:bar;" >
<div>
<span >
<span class="y" >AAA</span> <span class="x" >BBB</span> <span class="x" >CCC</span>
</span>
</div>
<div id="section-1-content-block-2-subheader" class="subheader_foo" subheader="baz" style="subheader:bar;" >
<span >
<span class="yy" >aaa</span> <span class="xx" >bbb</span> <span class="xx" >ccc</span>
</span>
</div>
</div>
<table
id="section-1-content-block-2-body"
class="body_foo" body="baz" style="body:bar;"
data-toggle="table"
>
<thead hidden>
<tr>
<th>
</th>
<th>
</th>
</tr>
</thead>
<tbody>
<tr>
<td id="section-1-content-block-2-cell-1-1" ><div class="show-scrollbars">Mean</div></td><td id="section-1-content-block-2-cell-1-2" ><div class="show-scrollbars">446</div></td></tr><tr>
<td id="section-1-content-block-2-cell-2-1" ><div class="show-scrollbars">Minimum</div></td><td id="section-1-content-block-2-cell-2-2" ><div class="show-scrollbars">1</div></td></tr></tbody>
</table>
</div>""".replace(" ", "").replace("\t", "").replace("\n", ""))
def render(cls, index_links_dict):
sections = []
cta_object = index_links_dict.pop("cta_object", None)
try:
content_blocks = []
# site name header
site_name_header_block = RenderedHeaderContent(
**{
"content_block_type":
"header",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$title_prefix | $site_name",
"params": {
"site_name": index_links_dict.get(
"site_name"),
"title_prefix": "Data Docs"
},
"styling": {
"params": {
"title_prefix": {
"tag": "strong"
}
}
},
}
}),
"styling": {
"classes": ["col-12", "ge-index-page-site-name-title"],
"header": {
"classes": ["alert", "alert-secondary"]
}
}
})
content_blocks.append(site_name_header_block)
table_rows = []
table_header_row = []
link_list_keys_to_render = []
header_dict = OrderedDict(
[["expectations_links", "Expectation Suite"],
["validations_links", "Validation Results (run_id)"]])
for link_lists_key, header in header_dict.items():
if index_links_dict.get(link_lists_key):
class_header_str = link_lists_key.replace("_", "-")
class_str = "ge-index-page-table-{}-header".format(
class_header_str)
header = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": header,
"params": {},
"styling": {
"classes": [class_str],
}
}
})
table_header_row.append(header)
link_list_keys_to_render.append(link_lists_key)
generator_table = RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"classes": ["col-12", "ge-index-page-table-container"],
"styles": {
"margin-top": "10px"
},
"body": {
"classes": [
"table", "table-sm",
"ge-index-page-generator-table"
]
}
}
})
table_rows += cls._generate_links_table_rows(
index_links_dict,
link_list_keys_to_render=link_list_keys_to_render)
content_blocks.append(generator_table)
if index_links_dict.get("profiling_links"):
profiling_table_rows = []
for profiling_link_dict in index_links_dict.get(
"profiling_links"):
profiling_table_rows.append([
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$link_text",
"params": {
"link_text":
profiling_link_dict[
"expectation_suite_name"] + "." +
profiling_link_dict["batch_identifier"]
},
"tag": "a",
"styling": {
"attributes": {
"href":
profiling_link_dict["filepath"]
},
"classes": [
"ge-index-page-table-expectation-suite-link"
]
}
},
})
])
content_blocks.append(
RenderedTableContent(
**{
"content_block_type": "table",
"header_row": ["Profiling Results"],
"table": profiling_table_rows,
"styling": {
"classes":
["col-12", "ge-index-page-table-container"],
"styles": {
"margin-top": "10px"
},
"body": {
"classes": [
"table", "table-sm",
"ge-index-page-generator-table"
]
}
}
}))
section = RenderedSectionContent(
**{
"section_name": index_links_dict.get("site_name"),
"content_blocks": content_blocks
})
sections.append(section)
index_page_document = RenderedDocumentContent(
**{
"renderer_type": "SiteIndexPageRenderer",
"utm_medium": "index-page",
"sections": sections
})
if cta_object:
index_page_document.cta_footer = CallToActionRenderer.render(
cta_object)
return index_page_document
except Exception as e:
exception_message = f'''\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
'''
exception_traceback = traceback.format_exc()
exception_message += f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
logger.error(exception_message, e, exc_info=True)
def render(cls, index_links_dict):
sections = []
cta_object = index_links_dict.pop("cta_object", None)
try:
content_blocks = []
# site name header
site_name_header_block = RenderedHeaderContent(
**{
"content_block_type":
"header",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$title_prefix | $site_name",
"params": {
"site_name": index_links_dict.get(
"site_name"),
"title_prefix": "Data Docs"
},
"styling": {
"params": {
"title_prefix": {
"tag": "strong"
}
}
},
}
}),
"styling": {
"classes": ["col-12", "ge-index-page-site-name-title"],
"header": {
"classes": ["alert", "alert-secondary"]
}
}
})
content_blocks.append(site_name_header_block)
table_rows = []
table_header_row = []
link_list_keys_to_render = []
header_dict = OrderedDict(
[["expectations_links", "Expectation Suite"],
["validations_links", "Validation Results (run_id)"]])
for link_lists_key, header in header_dict.items():
if index_links_dict.get(link_lists_key):
class_header_str = link_lists_key.replace("_", "-")
class_str = "ge-index-page-table-{}-header".format(
class_header_str)
header = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": header,
"params": {},
"styling": {
"classes": [class_str],
}
}
})
table_header_row.append(header)
link_list_keys_to_render.append(link_lists_key)
generator_table = RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"classes": ["col-12", "ge-index-page-table-container"],
"styles": {
"margin-top": "10px"
},
"body": {
"classes": [
"table", "table-sm",
"ge-index-page-generator-table"
]
}
}
})
table_rows += cls._generate_links_table_rows(
index_links_dict,
link_list_keys_to_render=link_list_keys_to_render)
content_blocks.append(generator_table)
if index_links_dict.get("profiling_links"):
profiling_table_rows = []
for profiling_link_dict in index_links_dict.get(
"profiling_links"):
profiling_table_rows.append([
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$link_text",
"params": {
"link_text":
profiling_link_dict[
"expectation_suite_name"] + "." +
profiling_link_dict["batch_identifier"]
},
"tag": "a",
"styling": {
"attributes": {
"href":
profiling_link_dict["filepath"]
},
"classes": [
"ge-index-page-table-expectation-suite-link"
]
}
},
})
])
content_blocks.append(
RenderedTableContent(
**{
"content_block_type": "table",
"header_row": ["Profiling Results"],
"table": profiling_table_rows,
"styling": {
"classes":
["col-12", "ge-index-page-table-container"],
"styles": {
"margin-top": "10px"
},
"body": {
"classes": [
"table", "table-sm",
"ge-index-page-generator-table"
]
}
}
}))
section = RenderedSectionContent(
**{
"section_name": index_links_dict.get("site_name"),
"content_blocks": content_blocks
})
sections.append(section)
index_page_document = RenderedDocumentContent(
**{
"renderer_type": "SiteIndexPageRenderer",
"utm_medium": "index-page",
"sections": sections
})
if cta_object:
index_page_document.cta_footer = CallToActionRenderer.render(
cta_object)
return index_page_document
except Exception as e:
logger.error("Exception occurred during data docs rendering: ",
e,
exc_info=True)
def _render_stats_table(cls, evrs):
table_rows = []
mean_evr = cls._find_evr_by_type(evrs,
"expect_column_mean_to_be_between")
if not mean_evr or mean_evr.exception_info["raised_exception"]:
return
mean_value = "{:.2f}".format(
mean_evr.result['observed_value']) if mean_evr else None
if mean_value:
table_rows.append([{
"content_block_type": "string_template",
"string_template": {
"template": "Mean",
"tooltip": {
"content": "expect_column_mean_to_be_between"
}
}
}, mean_value])
min_evr = cls._find_evr_by_type(evrs,
"expect_column_min_to_be_between")
min_value = "{:.2f}".format(
min_evr.result['observed_value']) if min_evr else None
if min_value:
table_rows.append([
{
"content_block_type": "string_template",
"string_template": {
"template": "Minimum",
"tooltip": {
"content": "expect_column_min_to_be_between"
}
}
},
min_value,
])
max_evr = cls._find_evr_by_type(evrs,
"expect_column_max_to_be_between")
max_value = "{:.2f}".format(
max_evr.result['observed_value']) if max_evr else None
if max_value:
table_rows.append([{
"content_block_type": "string_template",
"string_template": {
"template": "Maximum",
"tooltip": {
"content": "expect_column_max_to_be_between"
}
}
}, max_value])
if len(table_rows) > 0:
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": 'Statistics',
"tag": "h6"
}
}),
"table":
table_rows,
"styling": {
"classes": ["col-3", "mt-1", "pl-1", "pr-1"],
"body": {
"classes":
["table", "table-sm", "table-unbordered"],
}
},
})
else:
return
def _get_unexpected_table(cls, evr):
try:
result = evr.result
except KeyError:
return None
if result is None:
return None
if not result.get("partial_unexpected_list") and not result.get(
"partial_unexpected_counts"
):
return None
table_rows = []
if result.get("partial_unexpected_counts"):
# We will check to see whether we have *all* of the unexpected values
# accounted for in our count, and include counts if we do. If we do not,
# we will use this as simply a better (non-repeating) source of
# "sampled" unexpected values
total_count = 0
for unexpected_count_dict in result.get("partial_unexpected_counts"):
if not isinstance(unexpected_count_dict, dict):
# handles case: "partial_exception_counts requires a hashable type"
# this case is also now deprecated (because the error is moved to an errors key
# the error also *should have* been updated to "partial_unexpected_counts ..." long ago.
# NOTE: JPC 20200724 - Consequently, this codepath should be removed by approximately Q1 2021
continue
value = unexpected_count_dict.get("value")
count = unexpected_count_dict.get("count")
total_count += count
if value is not None and value != "":
table_rows.append([value, count])
elif value == "":
table_rows.append(["EMPTY", count])
else:
table_rows.append(["null", count])
# Check to see if we have *all* of the unexpected values accounted for. If so,
# we show counts. If not, we only show "sampled" unexpected values.
if total_count == result.get("unexpected_count"):
header_row = ["Unexpected Value", "Count"]
else:
header_row = ["Sampled Unexpected Values"]
table_rows = [[row[0]] for row in table_rows]
else:
header_row = ["Sampled Unexpected Values"]
sampled_values_set = set()
for unexpected_value in result.get("partial_unexpected_list"):
if unexpected_value:
string_unexpected_value = unexpected_value
elif unexpected_value == "":
string_unexpected_value = "EMPTY"
else:
string_unexpected_value = "null"
if string_unexpected_value not in sampled_values_set:
table_rows.append([string_unexpected_value])
sampled_values_set.add(string_unexpected_value)
unexpected_table_content_block = RenderedTableContent(
**{
"content_block_type": "table",
"table": table_rows,
"header_row": header_row,
"styling": {
"body": {"classes": ["table-bordered", "table-sm", "mt-3"]}
},
}
)
return unexpected_table_content_block
def _get_unexpected_table(cls, evr):
try:
result = evr.result
except KeyError:
return None
if result is None:
return None
if not result.get("partial_unexpected_list") and not result.get(
"partial_unexpected_counts"):
return None
table_rows = []
if result.get("partial_unexpected_counts"):
header_row = ["Unexpected Value", "Count"]
for unexpected_count in result.get("partial_unexpected_counts"):
if not isinstance(unexpected_count, dict):
# handles case: "partial_exception_counts requires a hashable type"
# this case is also now deprecated (because the error is moved to an errors key
# the error also *should have* been updated to "partial_unexpected_counts ..." long ago.
# NOTE: JPC 20200724 - Consequently, this codepath should be removed by approximately Q1 2021
continue
elif unexpected_count.get("value"):
table_rows.append([
unexpected_count.get("value"),
unexpected_count.get("count")
])
elif unexpected_count.get("value") == "":
table_rows.append(["EMPTY", unexpected_count.get("count")])
elif unexpected_count.get("value") is not None:
table_rows.append([
unexpected_count.get("value"),
unexpected_count.get("count")
])
else:
table_rows.append(["null", unexpected_count.get("count")])
else:
header_row = ["Unexpected Value"]
for unexpected_value in result.get("partial_unexpected_list"):
if unexpected_value:
table_rows.append([unexpected_value])
elif unexpected_value == "":
table_rows.append(["EMPTY"])
elif unexpected_value is not None:
table_rows.append([unexpected_value])
else:
table_rows.append(["null"])
unexpected_table_content_block = RenderedTableContent(
**{
"content_block_type": "table",
"table": table_rows,
"header_row": header_row,
"styling": {
"body": {
"classes": ["table-bordered", "table-sm", "mt-3"]
}
},
})
return unexpected_table_content_block
def test_render_section_page():
section = RenderedSectionContent(
**{
"section_name":
None,
"content_blocks": [
RenderedHeaderContent(**{
"content_block_type": "header",
"header": "Overview",
}),
RenderedTableContent(
**{
"content_block_type":
"table",
"header":
"Dataset info",
"table": [
["Number of variables", "12"],
["Number of observations", "891"],
],
"styling": {
"classes": ["col-6", "table-responsive"],
"styles": {
"margin-top": "20px"
},
"body": {
"classes": ["table", "table-sm"]
},
},
}),
],
}).to_json_dict()
rendered_doc = ge.render.view.view.DefaultJinjaSectionView().render({
"section":
section,
"section_loop": {
"index": 1
},
}) # .replace(" ", "").replace("\t", "").replace("\n", "")
print(rendered_doc)
rendered_doc = rendered_doc.replace(" ", "").replace("\t",
"").replace("\n", "")
assert (
rendered_doc ==
"""<div id="section-1" class="ge-section container-fluid mb-1 pb-1 pl-sm-3 px-0">
<div class="row" >
<div id="content-block-1" >
<div id="content-block-1-header" >
<h5>
Overview
</h5>
</div>
</div>
<div id="content-block-2" class="col-6 table-responsive" style="margin-top:20px;" >
<div id="content-block-2-header" >
<h5>
Dataset info
</h5>
</div>
<table
id="content-block-2-body"
class="table table-sm"
data-toggle="table"
>
<thead hidden>
<tr>
<th>
</th>
<th>
</th>
</tr>
</thead>
<tbody>
<tr>
<td id="content-block-2-cell-1-1" ><div class="show-scrollbars">Number of variables</div></td><td id="content-block-2-cell-1-2" ><div class="show-scrollbars">12</div></td></tr><tr>
<td id="content-block-2-cell-2-1" ><div class="show-scrollbars">Number of observations</div></td><td id="content-block-2-cell-2-2" ><div class="show-scrollbars">891</div></td></tr></tbody>
</table>
</div>
</div>
</div>""".replace(" ", "").replace("\t", "").replace("\n", ""))
def _prescriptive_renderer(cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration["kwargs"],
["column", "quantile_ranges", "row_condition", "condition_parser"],
)
template_str = "quantiles must be within the following value ranges."
if include_column_name:
template_str = "$column " + template_str
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = (conditional_template_str + ", then " +
template_str[0].lower() + template_str[1:])
params.update(conditional_params)
expectation_string_obj = {
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params
},
}
quantiles = params["quantile_ranges"]["quantiles"]
value_ranges = params["quantile_ranges"]["value_ranges"]
table_header_row = ["Quantile", "Min Value", "Max Value"]
table_rows = []
quantile_strings = {0.25: "Q1", 0.75: "Q3", 0.50: "Median"}
for quantile, value_range in zip(quantiles, value_ranges):
quantile_string = quantile_strings.get(quantile,
"{:3.2f}".format(quantile))
table_rows.append([
quantile_string,
str(value_range[0]) if value_range[0] is not None else "Any",
str(value_range[1]) if value_range[1] is not None else "Any",
])
quantile_range_table = RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"body": {
"classes": [
"table",
"table-sm",
"table-unbordered",
"col-4",
"mt-2",
],
},
"parent": {
"styles": {
"list-style-type": "none"
}
},
},
})
return [expectation_string_obj, quantile_range_table]
