def _get_status_icon(cls, evr):
if evr.exception_info["raised_exception"]:
return RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$icon",
"params": {
"icon": ""
},
"styling": {
"params": {
"icon": {
"classes": [
"fas", "fa-exclamation-triangle",
"text-warning"
],
"tag":
"i"
}
}
}
}
})
if evr.success:
return RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$icon",
"params": {
"icon": ""
},
"styling": {
"params": {
"icon": {
"classes":
["fas", "fa-check-circle", "text-success"],
"tag":
"i"
}
}
}
},
"styling": {
"parent": {
"classes":
["hide-succeeded-validation-target-child"]
}
}
})
else:
return RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$icon",
"params": {
"icon": ""
},
"styling": {
"params": {
"icon": {
"tag": "i",
"classes":
["fas", "fa-times", "text-danger"]
}
}
}
}
})
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column_list",
"ignore_row_if",
"row_condition",
"condition_parser",
"mostly",
],
)
if params["mostly"] is not None:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
mostly_str = ("" if params.get("mostly") is None else
", at least $mostly_pct % of the time")
template_str = f"Values must always be unique across columns{mostly_str}: "
for idx in range(len(params["column_list"]) - 1):
template_str += "$column_list_" + str(idx) + ", "
params["column_list_" + str(idx)] = params["column_list"][idx]
last_idx = len(params["column_list"]) - 1
template_str += "$column_list_" + str(last_idx)
params["column_list_" +
str(last_idx)] = params["column_list"][last_idx]
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = (conditional_template_str + ", then " +
template_str[0].lower() + template_str[1:])
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _prescriptive_renderer(
cls,
configuration: ExpectationConfiguration = None,
result: ExpectationValidationResult = None,
language: str = None,
runtime_configuration: dict = None,
**kwargs,
) -> List[Union[dict, str, RenderedStringTemplateContent,
RenderedTableContent, RenderedBulletListContent,
RenderedGraphContent, Any, ]]:
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"min_value",
"max_value",
"mostly",
"row_condition",
"condition_parser",
"strict_min",
"strict_max",
],
)
if (params["min_value"] is None) and (params["max_value"] is None):
template_str = "values may have any length."
else:
at_least_str, at_most_str = handle_strict_min_max(params)
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
if params["min_value"] is not None and params[
"max_value"] is not None:
template_str = f"values must be {at_least_str} $min_value and {at_most_str} $max_value characters long, at least $mostly_pct % of the time."
elif params["min_value"] is None:
template_str = f"values must be {at_most_str} $max_value characters long, at least $mostly_pct % of the time."
elif params["max_value"] is None:
template_str = f"values must be {at_least_str} $min_value characters long, at least $mostly_pct % of the time."
else:
if params["min_value"] is not None and params[
"max_value"] is not None:
template_str = f"values must always be {at_least_str} $min_value and {at_most_str} $max_value characters long."
elif params["min_value"] is None:
template_str = f"values must always be {at_most_str} $max_value characters long."
elif params["max_value"] is None:
template_str = f"values must always be {at_least_str} $min_value characters long."
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"min_value",
"max_value",
"row_condition",
"condition_parser",
"strict_min",
"strict_max",
],
)
if (params["min_value"] is None) and (params["max_value"] is None):
template_str = "discrete entropy may have any numerical value."
else:
at_least_str, at_most_str = handle_strict_min_max(params)
if params["min_value"] is not None and params[
"max_value"] is not None:
template_str = f"discrete entropy must be {at_least_str} $min_value and {at_most_str} $max_value."
elif params["min_value"] is None:
template_str = f"discrete entropy must be {at_most_str} $max_value."
elif params["max_value"] is None:
template_str = f"discrete entropy must be {at_least_str} $min_value."
if include_column_name:
template_str = "$column " + template_str
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = conditional_template_str + ", then " + template_str
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _generate_links_table_rows(cls, index_links_dict, link_list_keys_to_render):
section_rows = []
column_count = len(link_list_keys_to_render)
validations_links = index_links_dict.get("validations_links")
expectations_links = index_links_dict.get("expectations_links")
if column_count:
cell_width_pct = 100.0 / column_count
if "expectations_links" in link_list_keys_to_render:
for expectation_suite_link_dict in expectations_links:
expectation_suite_row = []
expectation_suite_name = expectation_suite_link_dict[
"expectation_suite_name"
]
expectation_suite_link = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$link_text",
"params": {"link_text": expectation_suite_name},
"tag": "a",
"styling": {
"attributes": {
"href": urllib.parse.quote(
expectation_suite_link_dict["filepath"]
)
},
"classes": [
"ge-index-page-table-expectation-suite-link"
],
},
},
"styling": {
"parent": {
"styles": {"width": "{}%".format(cell_width_pct),}
}
},
}
)
expectation_suite_row.append(expectation_suite_link)
if "validations_links" in link_list_keys_to_render:
sorted_validations_links = [
link_dict
for link_dict in sorted(
validations_links, key=lambda x: x["run_id"], reverse=True
)
if link_dict["expectation_suite_name"] == expectation_suite_name
]
validation_link_bullets = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "${validation_success} $link_text",
"params": {
"link_text": link_dict["run_id"],
"validation_success": "",
},
"tag": "a",
"styling": {
"attributes": {
"href": urllib.parse.quote(
link_dict["filepath"]
)
},
"params": {
"validation_success": {
"tag": "i",
"classes": [
"fas",
"fa-check-circle",
"text-success",
]
if link_dict["validation_success"]
else ["fas", "fa-times", "text-danger"],
}
},
"classes": [
"ge-index-page-table-validation-links-item"
],
},
},
"styling": {
"parent": {
"classes": ["hide-succeeded-validation-target"]
if link_dict["validation_success"]
else []
}
},
}
)
for link_dict in sorted_validations_links
if link_dict["expectation_suite_name"] == expectation_suite_name
]
validation_link_bullet_list = RenderedBulletListContent(
**{
"content_block_type": "bullet_list",
"bullet_list": validation_link_bullets,
"styling": {
"parent": {
"styles": {"width": "{}%".format(cell_width_pct)}
},
"body": {
"classes": [
"ge-index-page-table-validation-links-list"
]
},
},
}
)
expectation_suite_row.append(validation_link_bullet_list)
section_rows.append(expectation_suite_row)
if not expectations_links and "validations_links" in link_list_keys_to_render:
sorted_validations_links = [
link_dict
for link_dict in sorted(
validations_links, key=lambda x: x["run_id"], reverse=True
)
]
validation_link_bullets = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "${validation_success} $link_text",
"params": {
"link_text": link_dict["run_id"],
"validation_success": "",
},
"tag": "a",
"styling": {
"attributes": {
"href": urllib.parse.quote(link_dict["filepath"])
},
"params": {
"validation_success": {
"tag": "i",
"classes": [
"fas",
"fa-check-circle",
"text-success",
]
if link_dict["validation_success"]
else ["fas", "fa-times", "text-danger"],
}
},
"classes": [
"ge-index-page-table-validation-links-item"
],
},
},
"styling": {
"parent": {
"classes": ["hide-succeeded-validation-target"]
if link_dict["validation_success"]
else []
}
},
}
)
for link_dict in sorted_validations_links
]
validation_link_bullet_list = RenderedBulletListContent(
**{
"content_block_type": "bullet_list",
"bullet_list": validation_link_bullets,
"styling": {
"parent": {"styles": {"width": "{}%".format(cell_width_pct)}},
"body": {
"classes": ["ge-index-page-table-validation-links-list"]
},
},
}
)
section_rows.append([validation_link_bullet_list])
return section_rows
def _render_expectation_types(cls, evrs, content_blocks):
# NOTE: The evr-fetching function is an kinda similar to the code other_section_
# renderer.ProfilingResultsOverviewSectionRenderer._render_expectation_types
# type_counts = defaultdict(int)
# for evr in evrs:
# type_counts[evr.expectation_config.expectation_type] += 1
# bullet_list = sorted(type_counts.items(), key=lambda kv: -1*kv[1])
bullet_list = [{
"content_block_type": "string_template",
"string_template": {
"template": "$expectation_type $is_passing",
"params": {
"expectation_type":
evr.expectation_config.expectation_type,
"is_passing": str(evr.success),
},
"styling": {
"classes": [
"list-group-item",
"d-flex",
"justify-content-between",
"align-items-center",
],
"params": {
"is_passing": {
"classes":
["badge", "badge-secondary", "badge-pill"],
}
},
},
},
} for evr in evrs]
content_blocks.append(
RenderedBulletListContent(
**{
"content_block_type":
"bullet_list",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template":
'Expectation types <span class="mr-3 triangle"></span>',
"tag": "h6",
},
}),
"bullet_list":
bullet_list,
"styling": {
"classes": ["col-12", "mt-1"],
"header": {
"classes": ["collapsed"],
"attributes": {
"data-toggle": "collapse",
"href": "#{{content_block_id}}-body",
"role": "button",
"aria-expanded": "true",
"aria-controls": "collapseExample",
},
"styles": {
"cursor": "pointer",
},
},
"body": {
"classes": ["list-group", "collapse"],
},
},
}))
def _descriptive_example_values_block_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
assert result, "Must pass in result."
if "partial_unexpected_counts" in result.result:
partial_unexpected_counts = result.result[
"partial_unexpected_counts"]
values = [str(v["value"]) for v in partial_unexpected_counts]
elif "partial_unexpected_list" in result.result:
values = [
str(item) for item in result.result["partial_unexpected_list"]
]
else:
return
classes = ["col-3", "mt-1", "pl-1", "pr-1"]
if any(len(value) > 80 for value in values):
content_block_type = "bullet_list"
content_block_class = RenderedBulletListContent
else:
content_block_type = "value_list"
content_block_class = ValueListContent
new_block = content_block_class(
**{
"content_block_type":
content_block_type,
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Example Values",
"tooltip": {
"content": "expect_column_values_to_be_in_set"
},
"tag": "h6",
},
}),
content_block_type: [{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": value
},
"styling": {
"default": {
"classes": ["badge", "badge-info"]
if content_block_type == "value_list" else [],
"styles": {
"word-break": "break-all"
},
},
},
},
} for value in values],
"styling": {
"classes": classes,
},
})
return new_block
def _diagnostic_unexpected_statement_renderer(
cls,
configuration: ExpectationConfiguration = None,
result: ExpectationValidationResult = None,
language: str = None,
runtime_configuration: dict = None,
**kwargs,
):
assert result, "Must provide a result object."
success = result.success
result = result.result
if result.exception_info["raised_exception"]:
exception_message_template_str = (
"\n\n$expectation_type raised an exception:\n$exception_message"
)
exception_message = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": exception_message_template_str,
"params": {
"expectation_type":
result.expectation_config.expectation_type,
"exception_message":
result.exception_info["exception_message"],
},
"tag": "strong",
"styling": {
"classes": ["text-danger"],
"params": {
"exception_message": {
"tag": "code"
},
"expectation_type": {
"classes":
["badge", "badge-danger", "mb-2"]
},
},
},
},
})
exception_traceback_collapse = CollapseContent(
**{
"collapse_toggle_link":
"Show exception traceback...",
"collapse": [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template":
result.
exception_info["exception_traceback"],
"tag":
"code",
},
})
],
})
return [exception_message, exception_traceback_collapse]
if success or not result_dict.get("unexpected_count"):
return []
else:
unexpected_count = num_to_str(result_dict["unexpected_count"],
use_locale=True,
precision=20)
unexpected_percent = (
num_to_str(result_dict["unexpected_percent"], precision=4) +
"%")
element_count = num_to_str(result_dict["element_count"],
use_locale=True,
precision=20)
template_str = (
"\n\n$unexpected_count unexpected values found. "
"$unexpected_percent of $element_count total rows.")
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": {
"unexpected_count": unexpected_count,
"unexpected_percent": unexpected_percent,
"element_count": element_count,
},
"tag": "strong",
"styling": {
"classes": ["text-danger"]
},
},
})
]
def _render_validation_header(cls, validation_results):
success = validation_results.success
expectation_suite_name = validation_results.meta[
"expectation_suite_name"]
expectation_suite_path_components = (
[".." for _ in range(len(expectation_suite_name.split(".")) + 3)] +
["expectations"] + str(expectation_suite_name).split("."))
expectation_suite_path = (
os.path.join(*expectation_suite_path_components) + ".html")
# TODO: deprecate dual batch api support in 0.14
batch_kwargs = (validation_results.meta.get("batch_kwargs", {})
or validation_results.meta.get("batch_spec", {}) or {})
data_asset_name = batch_kwargs.get("data_asset_name")
if success:
success = "Succeeded"
html_success_icon = (
'<i class="fas fa-check-circle text-success" aria-hidden="true"></i>'
)
else:
success = "Failed"
html_success_icon = (
'<i class="fas fa-times text-danger" aria-hidden="true"></i>')
return RenderedHeaderContent(
**{
"content_block_type":
"header",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Overview",
"tag": "h5",
"styling": {
"classes": ["m-0"]
},
},
}),
"subheader":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template":
"${suite_title} ${expectation_suite_name}\n ${data_asset} ${data_asset_name}\n ${status_title} ${html_success_icon} ${success}",
"params": {
"suite_title": "Expectation Suite:",
"data_asset": "Data asset:",
"data_asset_name": data_asset_name,
"status_title": "Status:",
"expectation_suite_name":
expectation_suite_name,
"success": success,
"html_success_icon": html_success_icon,
},
"styling": {
"params": {
"suite_title": {
"classes": ["h6"]
},
"status_title": {
"classes": ["h6"]
},
"expectation_suite_name": {
"tag": "a",
"attributes": {
"href": expectation_suite_path
},
},
},
"classes": ["mb-0", "mt-1"],
},
},
}),
"styling": {
"classes": ["col-12", "p-0"],
"header": {
"classes": ["alert", "alert-secondary"]
},
},
})
def _render_values_set(cls, evrs):
set_evr = cls._find_evr_by_type(evrs,
"expect_column_values_to_be_in_set")
if not set_evr or set_evr.exception_info["raised_exception"]:
return
if set_evr and "partial_unexpected_counts" in set_evr.result:
partial_unexpected_counts = set_evr.result[
"partial_unexpected_counts"]
values = [str(v["value"]) for v in partial_unexpected_counts]
elif set_evr and "partial_unexpected_list" in set_evr.result:
values = [
str(item) for item in set_evr.result["partial_unexpected_list"]
]
else:
return
classes = ["col-3", "mt-1", "pl-1", "pr-1"]
if any(len(value) > 80 for value in values):
content_block_type = "bullet_list"
content_block_class = RenderedBulletListContent
else:
content_block_type = "value_list"
content_block_class = ValueListContent
new_block = content_block_class(
**{
"content_block_type":
content_block_type,
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Example Values",
"tooltip": {
"content": "expect_column_values_to_be_in_set"
},
"tag": "h6"
}
}),
content_block_type: [{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": value
},
"styling": {
"default": {
"classes": ["badge", "badge-info"]
if content_block_type == "value_list" else [],
"styles": {
"word-break": "break-all"
}
},
}
}
} for value in values],
"styling": {
"classes": classes,
}
})
return new_block
def _render_bar_chart_table(cls, evrs):
distinct_values_set_evr = cls._find_evr_by_type(
evrs, "expect_column_distinct_values_to_be_in_set")
if not distinct_values_set_evr or distinct_values_set_evr.exception_info[
"raised_exception"]:
return
value_count_dicts = distinct_values_set_evr.result['details'][
'value_counts']
if isinstance(value_count_dicts, pd.Series):
values = value_count_dicts.index.tolist()
counts = value_count_dicts.tolist()
else:
values = [
value_count_dict['value']
for value_count_dict in value_count_dicts
]
counts = [
value_count_dict['count']
for value_count_dict in value_count_dicts
]
df = pd.DataFrame({
"value": values,
"count": counts,
})
if len(values) > 60:
return None
else:
chart_pixel_width = (len(values) / 60.0) * 500
if chart_pixel_width < 250:
chart_pixel_width = 250
chart_container_col_width = round((len(values) / 60.0) * 6)
if chart_container_col_width < 4:
chart_container_col_width = 4
elif chart_container_col_width >= 5:
chart_container_col_width = 6
elif chart_container_col_width >= 4:
chart_container_col_width = 5
mark_bar_args = {}
if len(values) == 1:
mark_bar_args["size"] = 20
bars = alt.Chart(df).mark_bar(**mark_bar_args).encode(
y='count:Q', x="value:O",
tooltip=["value", "count"]).properties(height=400,
width=chart_pixel_width,
autosize="fit")
chart = bars.to_json()
new_block = RenderedGraphContent(
**{
"content_block_type":
"graph",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Value Counts",
"tooltip": {
"content":
"expect_column_distinct_values_to_be_in_set"
},
"tag": "h6"
}
}),
"graph":
chart,
"styling": {
"classes":
["col-" + str(chart_container_col_width), "mt-1"],
}
})
return new_block
def _render_stats_table(cls, evrs):
table_rows = []
mean_evr = cls._find_evr_by_type(evrs,
"expect_column_mean_to_be_between")
if not mean_evr or mean_evr.exception_info["raised_exception"]:
return
mean_value = "{:.2f}".format(
mean_evr.result['observed_value']) if mean_evr else None
if mean_value:
table_rows.append([{
"content_block_type": "string_template",
"string_template": {
"template": "Mean",
"tooltip": {
"content": "expect_column_mean_to_be_between"
}
}
}, mean_value])
min_evr = cls._find_evr_by_type(evrs,
"expect_column_min_to_be_between")
min_value = "{:.2f}".format(
min_evr.result['observed_value']) if min_evr else None
if min_value:
table_rows.append([
{
"content_block_type": "string_template",
"string_template": {
"template": "Minimum",
"tooltip": {
"content": "expect_column_min_to_be_between"
}
}
},
min_value,
])
max_evr = cls._find_evr_by_type(evrs,
"expect_column_max_to_be_between")
max_value = "{:.2f}".format(
max_evr.result['observed_value']) if max_evr else None
if max_value:
table_rows.append([{
"content_block_type": "string_template",
"string_template": {
"template": "Maximum",
"tooltip": {
"content": "expect_column_max_to_be_between"
}
}
}, max_value])
if len(table_rows) > 0:
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": 'Statistics',
"tag": "h6"
}
}),
"table":
table_rows,
"styling": {
"classes": ["col-3", "mt-1", "pl-1", "pr-1"],
"body": {
"classes":
["table", "table-sm", "table-unbordered"],
}
},
})
else:
return
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get("include_column_name", True)
include_column_name = (
include_column_name if include_column_name is not None else True
)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"value_set",
"mostly",
"parse_strings_as_datetimes",
"row_condition",
"condition_parser",
],
)
if params["value_set"] is None or len(params["value_set"]) == 0:
values_string = "[ ]"
else:
for i, v in enumerate(params["value_set"]):
params["v__" + str(i)] = v
values_string = " ".join(
["$v__" + str(i) for i, v in enumerate(params["value_set"])]
)
template_str = "values must not belong to this set: " + values_string
if params["mostly"] is not None:
params["mostly_pct"] = num_to_str(
params["mostly"] * 100, precision=15, no_scientific=True
)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str += ", at least $mostly_pct % of the time."
else:
template_str += "."
if params.get("parse_strings_as_datetimes"):
template_str += " Values should be parsed as datetimes."
if include_column_name:
template_str = "$column " + template_str
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(params["row_condition"])
template_str = conditional_template_str + ", then " + template_str
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
}
)
]
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column", "mostly", "json_schema", "row_condition",
"condition_parser"
],
)
if not params.get("json_schema"):
template_str = "values must match a JSON Schema but none was specified."
else:
params[
"formatted_json"] = f"<pre>{json.dumps(params.get('json_schema'), indent=4)}</pre>"
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str = "values must match the following JSON Schema, at least $mostly_pct % of the time: $formatted_json"
else:
template_str = (
"values must match the following JSON Schema: $formatted_json"
)
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": {
"params": {
"formatted_json": {
"classes": []
}
}
},
},
})
]
def test_ValidationResultsTableContentBlockRenderer_get_unexpected_statement(
evr_success, evr_failed
):
evr_no_result = ExpectationValidationResult(
success=True,
exception_info={
"raised_exception": False,
"exception_message": None,
"exception_traceback": None,
},
expectation_config=ExpectationConfiguration(
expectation_type="expect_table_row_count_to_be_between",
kwargs={"min_value": 0, "max_value": None, "result_format": "SUMMARY"},
),
)
evr_failed_no_unexpected_count = ExpectationValidationResult(
success=False,
result={
"element_count": 1313,
"missing_count": 0,
"missing_percent": 0.0,
"unexpected_percent": 0.2284843869002285,
"unexpected_percent_nonmissing": 0.2284843869002285,
"partial_unexpected_list": [
"Daly, Mr Peter Denis ",
"Barber, Ms ",
"Geiger, Miss Emily ",
],
"partial_unexpected_index_list": [77, 289, 303],
"partial_unexpected_counts": [
{"value": "Barber, Ms ", "count": 1},
{"value": "Daly, Mr Peter Denis ", "count": 1},
{"value": "Geiger, Miss Emily ", "count": 1},
],
},
exception_info={
"raised_exception": False,
"exception_message": None,
"exception_traceback": None,
},
expectation_config=ExpectationConfiguration(
expectation_type="expect_column_values_to_not_match_regex",
kwargs={
"column": "Name",
"regex": "^\\s+|\\s+$",
"result_format": "SUMMARY",
},
),
)
# test for succeeded evr
output_1 = get_renderer_impl(
object_name=evr_success.expectation_config.expectation_type,
renderer_type="renderer.diagnostic.unexpected_statement",
)[1](result=evr_success)
assert output_1 == []
# test for failed evr
output_2 = get_renderer_impl(
object_name=evr_failed.expectation_config.expectation_type,
renderer_type="renderer.diagnostic.unexpected_statement",
)[1](result=evr_failed)
assert output_2 == [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "\n\n$unexpected_count unexpected values found. $unexpected_percent of $element_count total rows.",
"params": {
"unexpected_count": "3",
"unexpected_percent": "≈0.2285%",
"element_count": "1,313",
},
"tag": "strong",
"styling": {"classes": ["text-danger"]},
},
}
)
]
# test for evr with no "result" key
output_3 = get_renderer_impl(
object_name=evr_no_result.expectation_config.expectation_type,
renderer_type="renderer.diagnostic.unexpected_statement",
)[1](result=evr_no_result)
print(json.dumps(output_3, indent=2))
assert output_3 == []
# test for evr with no unexpected count
output_4 = get_renderer_impl(
object_name=evr_failed_no_unexpected_count.expectation_config.expectation_type,
renderer_type="renderer.diagnostic.unexpected_statement",
)[1](result=evr_failed_no_unexpected_count)
print(output_4)
assert output_4 == []
# test for evr with exception
evr_failed_exception = ExpectationValidationResult(
success=False,
exception_info={
"raised_exception": True,
"exception_message": "Unrecognized column: not_a_real_column",
"exception_traceback": "Traceback (most recent call last):\n...more_traceback...",
},
expectation_config=ExpectationConfiguration(
expectation_type="expect_column_values_to_not_match_regex",
kwargs={
"column": "Name",
"regex": "^\\s+|\\s+$",
"result_format": "SUMMARY",
},
),
)
output_5 = get_renderer_impl(
object_name=evr_failed_exception.expectation_config.expectation_type,
renderer_type="renderer.diagnostic.unexpected_statement",
)[1](result=evr_failed_exception)
output_5 = [content.to_json_dict() for content in output_5]
expected_output_5 = [
{
"content_block_type": "string_template",
"string_template": {
"template": "\n\n$expectation_type raised an exception:\n$exception_message",
"params": {
"expectation_type": "expect_column_values_to_not_match_regex",
"exception_message": "Unrecognized column: not_a_real_column",
},
"tag": "strong",
"styling": {
"classes": ["text-danger"],
"params": {
"exception_message": {"tag": "code"},
"expectation_type": {
"classes": ["badge", "badge-danger", "mb-2"]
},
},
},
},
},
{
"content_block_type": "collapse",
"collapse_toggle_link": "Show exception traceback...",
"collapse": [
{
"content_block_type": "string_template",
"string_template": {
"template": "Traceback (most recent call last):\n...more_traceback...",
"tag": "code",
},
}
],
"inline_link": False,
},
]
assert output_5 == expected_output_5
def _render_nested_table_from_dict(cls,
input_dict,
header=None,
sub_table=False):
table_rows = []
for kwarg, value in input_dict.items():
if not isinstance(value, (dict, OrderedDict)):
table_row = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(kwarg)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": ["pr-3"],
}
},
}),
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(value)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": [],
}
},
}),
]
else:
table_row = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$value",
"params": {
"value": str(kwarg)
},
"styling": {
"default": {
"styles": {
"word-break": "break-all"
}
},
},
},
"styling": {
"parent": {
"classes": ["pr-3"],
}
},
}),
cls._render_nested_table_from_dict(value, sub_table=True),
]
table_rows.append(table_row)
table_rows.sort(
key=lambda row: row[0].string_template["params"]["value"])
if sub_table:
return RenderedTableContent(
**{
"content_block_type": "table",
"table": table_rows,
"styling": {
"classes": ["col-6", "table-responsive"],
"body": {
"classes": ["table", "table-sm", "m-0"]
},
"parent": {
"classes": ["pt-0", "pl-0", "border-top-0"]
},
},
})
else:
return RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": header,
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"table":
table_rows,
"styling": {
"body": {
"classes": ["table", "table-sm"],
"styles": {
"margin-bottom": "0.5rem !important",
"margin-top": "0.5rem !important",
},
}
},
})
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
["column", "regex", "mostly", "row_condition", "condition_parser"],
)
if not params.get("regex"):
template_str = (
"values must match a regular expression but none was specified."
)
else:
template_str = "values must match this regular expression: $regex"
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str += ", at least $mostly_pct % of the time."
else:
template_str += "."
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
params_with_json_schema = {
"column": {
"schema": {
"type": "string"
},
"value": params.get("column")
},
"mostly": {
"schema": {
"type": "number"
},
"value": params.get("mostly")
},
"mostly_pct": {
"schema": {
"type": "number"
},
"value": params.get("mostly_pct"),
},
"regex": {
"schema": {
"type": "string"
},
"value": params.get("regex")
},
"row_condition": {
"schema": {
"type": "string"
},
"value": params.get("row_condition"),
},
"condition_parser": {
"schema": {
"type": "string"
},
"value": params.get("condition_parser"),
},
}
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _render_expectation_suite_notes(cls, expectations):
content = []
total_expectations = len(expectations.expectations)
columns = []
for exp in expectations.expectations:
if "column" in exp.kwargs:
columns.append(exp.kwargs["column"])
total_columns = len(set(columns))
content += [
# TODO: Leaving these two paragraphs as placeholders for later development.
# "This Expectation suite was first generated by {BasicDatasetProfiler} on {date}, using version {xxx} of Great Expectations.",
# "{name}, {name}, and {name} have also contributed additions and revisions.",
"This Expectation suite currently contains %d total Expectations across %d columns."
% (
total_expectations,
total_columns,
),
]
if "notes" in expectations.meta:
notes = expectations.meta["notes"]
note_content = None
if isinstance(notes, str):
note_content = [notes]
elif isinstance(notes, list):
note_content = notes
elif isinstance(notes, dict):
if "format" in notes:
if notes["format"] == "string":
if isinstance(notes["content"], str):
note_content = [notes["content"]]
elif isinstance(notes["content"], list):
note_content = notes["content"]
else:
logger.warning(
"Unrecognized Expectation suite notes format. Skipping rendering."
)
elif notes["format"] == "markdown":
if isinstance(notes["content"], str):
note_content = [
RenderedMarkdownContent(
**{
"content_block_type": "markdown",
"markdown": notes["content"],
"styling": {
"parent": {}
},
})
]
elif isinstance(notes["content"], list):
note_content = [
RenderedMarkdownContent(
**{
"content_block_type": "markdown",
"markdown": note,
"styling": {
"parent": {}
},
}) for note in notes["content"]
]
else:
logger.warning(
"Unrecognized Expectation suite notes format. Skipping rendering."
)
else:
logger.warning(
"Unrecognized Expectation suite notes format. Skipping rendering."
)
if note_content is not None:
content += note_content
return TextContent(
**{
"content_block_type":
"text",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "Notes",
"tag": "h6",
"styling": {
"classes": ["m-0"]
},
},
}),
"text":
content,
"styling": {
"classes": ["col-12", "table-responsive", "mt-1"],
"body": {
"classes": ["table", "table-sm"]
},
},
})
def _prescriptive_renderer(cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column", "mostly", "xml_schema", "row_condition",
"condition_parser"
],
)
if not params.get("xml_schema"):
template_str = "values must match a XML Schema but none was specified."
else:
params["formatted_xml"] = (
"<pre>" +
etree.tostring(params.get("xml_schema"), pretty_print=True) +
"</pre>" # TODO:
)
if params["mostly"] is not None:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str = "values must match the following XML Schema, at least $mostly_pct % of the time: $formatted_xml"
else:
template_str = (
"values must match the following XML Schema: $formatted_xml"
)
if include_column_name:
template_str = "$column " + template_str
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = conditional_template_str + ", then " + template_str
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": {
"params": {
"formatted_xml": {
"classes": []
}
}
},
},
})
]
def _prescriptive_renderer(cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column", "type_list", "mostly", "row_condition",
"condition_parser"
],
)
if params["type_list"] is not None:
for i, v in enumerate(params["type_list"]):
params["v__" + str(i)] = v
values_string = " ".join(
["$v__" + str(i) for i, v in enumerate(params["type_list"])])
if params["mostly"] is not None:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
if include_column_name:
template_str = (
"$column value types must belong to this set: " +
values_string +
", at least $mostly_pct % of the time.")
else:
template_str = ("value types must belong to this set: " +
values_string +
", at least $mostly_pct % of the time.")
else:
if include_column_name:
template_str = (
"$column value types must belong to this set: " +
values_string + ".")
else:
template_str = ("value types must belong to this set: " +
values_string + ".")
else:
if include_column_name:
template_str = "$column value types may be any value, but observed value will be reported"
else:
template_str = (
"value types may be any value, but observed value will be reported"
)
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = conditional_template_str + ", then " + template_str
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"value_set",
"parse_strings_as_datetimes",
"row_condition",
"condition_parser",
],
)
if params["value_set"] is None or len(params["value_set"]) == 0:
values_string = "[ ]"
else:
for i, v in enumerate(params["value_set"]):
params["v__" + str(i)] = v
values_string = " ".join(
["$v__" + str(i) for i, v in enumerate(params["value_set"])])
template_str = "distinct values must contain this set: " + values_string + "."
if params.get("parse_strings_as_datetimes"):
template_str += " Values should be parsed as datetimes."
if include_column_name:
template_str = "$column " + template_str
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = conditional_template_str + ", then " + template_str
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def test_rendering_components_with_styling():
# Medium-complicated example to verify that all the things are correctly piped to all the places
header_component_content = RenderedTableContent(
**{
"content_block_type":
"table",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$var1 $var2 $var3",
"params": {
"var1": "AAA",
"var2": "BBB",
"var3": "CCC",
},
"styling": {
"default": {
"classes": ["x"]
},
"params": {
"var1": {
"classes": ["y"]
}
},
},
},
}),
"subheader":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$var1 $var2 $var3",
"params": {
"var1": "aaa",
"var2": "bbb",
"var3": "ccc",
},
"styling": {
"default": {
"classes": ["xx"]
},
"params": {
"var1": {
"classes": ["yy"]
}
},
},
},
}),
"table": [
["Mean", "446"],
["Minimum", "1"],
],
"styling": {
"classes": ["root_foo"],
"styles": {
"root": "bar"
},
"attributes": {
"root": "baz"
},
"header": {
"classes": ["header_foo"],
"styles": {
"header": "bar"
},
"attributes": {
"header": "baz"
},
},
"subheader": {
"classes": ["subheader_foo"],
"styles": {
"subheader": "bar"
},
"attributes": {
"subheader": "baz"
},
},
"body": {
"classes": ["body_foo"],
"styles": {
"body": "bar"
},
"attributes": {
"body": "baz"
},
},
},
}).to_json_dict()
rendered_doc = ge.render.view.view.DefaultJinjaComponentView().render({
"content_block":
header_component_content,
"section_loop": {
"index": 1
},
"content_block_loop": {
"index": 2
},
})
print(rendered_doc)
rendered_doc = rendered_doc.replace(" ", "").replace("\t",
"").replace("\n", "")
assert (rendered_doc == """
<div id="section-1-content-block-2" class="root_foo" root="baz" style="root:bar;" >
<div id="section-1-content-block-2-header" class="header_foo" header="baz" style="header:bar;" >
<div>
<span >
<span class="y" >AAA</span> <span class="x" >BBB</span> <span class="x" >CCC</span>
</span>
</div>
<div id="section-1-content-block-2-subheader" class="subheader_foo" subheader="baz" style="subheader:bar;" >
<span >
<span class="yy" >aaa</span> <span class="xx" >bbb</span> <span class="xx" >ccc</span>
</span>
</div>
</div>
<table
id="section-1-content-block-2-body"
class="body_foo" body="baz" style="body:bar;"
data-toggle="table"
>
<thead hidden>
<tr>
<th>
</th>
<th>
</th>
</tr>
</thead>
<tbody>
<tr>
<td id="section-1-content-block-2-cell-1-1" ><div class="show-scrollbars">Mean</div></td><td id="section-1-content-block-2-cell-1-2" ><div class="show-scrollbars">446</div></td></tr><tr>
<td id="section-1-content-block-2-cell-2-1" ><div class="show-scrollbars">Minimum</div></td><td id="section-1-content-block-2-cell-2-2" ><div class="show-scrollbars">1</div></td></tr></tbody>
</table>
</div>""".replace(" ", "").replace("\t", "").replace("\n", ""))
def render(cls, index_links_dict):
sections = []
cta_object = index_links_dict.pop("cta_object", None)
try:
content_blocks = []
# site name header
site_name_header_block = RenderedHeaderContent(
**{
"content_block_type": "header",
"header": RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$title_prefix | $site_name",
"params": {
"site_name": index_links_dict.get("site_name"),
"title_prefix": "Data Docs",
},
"styling": {
"params": {"title_prefix": {"tag": "strong"}}
},
},
}
),
"styling": {
"classes": ["col-12", "ge-index-page-site-name-title"],
"header": {"classes": ["alert", "alert-secondary"]},
},
}
)
content_blocks.append(site_name_header_block)
table_rows = []
table_header_row = []
link_list_keys_to_render = []
header_dict = OrderedDict(
[
["expectations_links", "Expectation Suite"],
["validations_links", "Validation Results (run_id)"],
]
)
for link_lists_key, header in header_dict.items():
if index_links_dict.get(link_lists_key):
class_header_str = link_lists_key.replace("_", "-")
class_str = "ge-index-page-table-{}-header".format(class_header_str)
header = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": header,
"params": {},
"styling": {"classes": [class_str],},
},
}
)
table_header_row.append(header)
link_list_keys_to_render.append(link_lists_key)
generator_table = RenderedTableContent(
**{
"content_block_type": "table",
"header_row": table_header_row,
"table": table_rows,
"styling": {
"classes": ["col-12", "ge-index-page-table-container"],
"styles": {"margin-top": "10px"},
"body": {
"classes": [
"table",
"table-sm",
"ge-index-page-generator-table",
]
},
},
}
)
table_rows += cls._generate_links_table_rows(
index_links_dict, link_list_keys_to_render=link_list_keys_to_render
)
content_blocks.append(generator_table)
if index_links_dict.get("profiling_links"):
profiling_table_rows = []
for profiling_link_dict in index_links_dict.get("profiling_links"):
profiling_table_rows.append(
[
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$link_text",
"params": {
"link_text": profiling_link_dict[
"expectation_suite_name"
]
+ "."
+ profiling_link_dict["batch_identifier"]
},
"tag": "a",
"styling": {
"attributes": {
"href": urllib.parse.quote(
profiling_link_dict["filepath"]
)
},
"classes": [
"ge-index-page-table-expectation-suite-link"
],
},
},
}
)
]
)
content_blocks.append(
RenderedTableContent(
**{
"content_block_type": "table",
"header_row": ["Profiling Results"],
"table": profiling_table_rows,
"styling": {
"classes": ["col-12", "ge-index-page-table-container"],
"styles": {"margin-top": "10px"},
"body": {
"classes": [
"table",
"table-sm",
"ge-index-page-generator-table",
]
},
},
}
)
)
section = RenderedSectionContent(
**{
"section_name": index_links_dict.get("site_name"),
"content_blocks": content_blocks,
}
)
sections.append(section)
index_page_document = RenderedDocumentContent(
**{
"renderer_type": "SiteIndexPageRenderer",
"utm_medium": "index-page",
"sections": sections,
}
)
if cta_object:
index_page_document.cta_footer = CallToActionRenderer.render(cta_object)
return index_page_document
except Exception as e:
exception_message = f"""\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
"""
exception_traceback = traceback.format_exc()
exception_message += (
f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
)
logger.error(exception_message, e, exc_info=True)
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column_A",
"column_B",
"parse_strings_as_datetimes",
"ignore_row_if",
"mostly",
"or_equal",
"row_condition",
"condition_parser",
],
)
if (params["column_A"] is None) or (params["column_B"] is None):
template_str = "$column has a bogus `expect_column_pair_values_A_to_be_greater_than_B` expectation."
params["row_condition"] = None
if params["mostly"] is None:
if params["or_equal"] in [None, False]:
template_str = "Values in $column_A must always be greater than those in $column_B."
else:
template_str = "Values in $column_A must always be greater than or equal to those in $column_B."
else:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
if params["or_equal"] in [None, False]:
template_str = "Values in $column_A must be greater than those in $column_B, at least $mostly_pct % of the time."
else:
template_str = "Values in $column_A must be greater than or equal to those in $column_B, at least $mostly_pct % of the time."
if params.get("parse_strings_as_datetimes"):
template_str += " Values should be parsed as datetimes."
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = (conditional_template_str + ", then " +
template_str[0].lower() + template_str[1:])
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get("include_column_name", True)
include_column_name = (
include_column_name if include_column_name is not None else True
)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"regex_list",
"mostly",
"match_on",
"row_condition",
"condition_parser",
],
)
if not params.get("regex_list") or len(params.get("regex_list")) == 0:
values_string = "[ ]"
else:
for i, v in enumerate(params["regex_list"]):
params[f"v__{str(i)}"] = v
values_string = " ".join(
[f"$v__{str(i)}" for i, v in enumerate(params["regex_list"])]
)
if params.get("match_on") == "all":
template_str = (
"values must match all of the following regular expressions: "
+ values_string
)
else:
template_str = (
"values must match any of the following regular expressions: "
+ values_string
)
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(
params["mostly"] * 100, precision=15, no_scientific=True
)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str += ", at least $mostly_pct % of the time."
else:
template_str += "."
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
}
)
]
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"min_value",
"max_value",
"mostly",
"row_condition",
"condition_parser",
"strict_min",
"strict_max",
],
)
template_str = ""
if (params["min_value"] is None) and (params["max_value"] is None):
template_str += "may have any numerical value."
else:
at_least_str, at_most_str = handle_strict_min_max(params)
mostly_str = ""
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
mostly_str = ", at least $mostly_pct % of the time"
if params["min_value"] is not None and params[
"max_value"] is not None:
template_str += f"values must be {at_least_str} $min_value and {at_most_str} $max_value{mostly_str}."
elif params["min_value"] is None:
template_str += f"values must be {at_most_str} $max_value{mostly_str}."
elif params["max_value"] is None:
template_str += f"values must be {at_least_str} $min_value{mostly_str}."
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def _render_expectation_types(cls, evrs, content_blocks):
type_counts = defaultdict(int)
for evr in evrs.results:
type_counts[evr.expectation_config.expectation_type] += 1
bullet_list_items = sorted(type_counts.items(),
key=lambda kv: -1 * kv[1])
bullet_list_items = [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$expectation_type $expectation_count",
"params": {
"expectation_type": tr[0],
"expectation_count": tr[1],
},
"styling": {
"classes": [
"list-group-item",
"d-flex",
"justify-content-between",
"align-items-center",
],
"params": {
"expectation_count": {
"classes": [
"badge",
"badge-secondary",
"badge-pill",
],
}
},
},
},
"styling": {
"parent": {
"styles": {
"list-style-type": "none"
}
}
},
}) for tr in bullet_list_items
]
bullet_list = RenderedBulletListContent(
**{
"content_block_type": "bullet_list",
"bullet_list": bullet_list_items,
"styling": {
"classes": ["col-12", "mt-1"],
"body": {
"classes": ["list-group"],
},
},
})
bullet_list_collapse = CollapseContent(
**{
"collapse_toggle_link": "Show Expectation Types...",
"collapse": [bullet_list],
"styling": {
"classes": ["col-12", "p-1"]
},
})
content_blocks.append(bullet_list_collapse)
def _prescriptive_renderer(
cls,
configuration=None,
result=None,
language=None,
runtime_configuration=None,
**kwargs,
):
runtime_configuration = runtime_configuration or {}
include_column_name = runtime_configuration.get(
"include_column_name", True)
include_column_name = (include_column_name
if include_column_name is not None else True)
styling = runtime_configuration.get("styling")
params = substitute_none_for_missing(
configuration.kwargs,
[
"column",
"strictly",
"mostly",
"parse_strings_as_datetimes",
"row_condition",
"condition_parser",
],
)
if params.get("strictly"):
template_str = "values must be strictly less than previous values"
else:
template_str = "values must be less than or equal to previous values"
if params["mostly"] is not None and params["mostly"] < 1.0:
params["mostly_pct"] = num_to_str(params["mostly"] * 100,
precision=15,
no_scientific=True)
# params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
template_str += ", at least $mostly_pct % of the time."
else:
template_str += "."
if params.get("parse_strings_as_datetimes"):
template_str += " Values should be parsed as datetimes."
if include_column_name:
template_str = f"$column {template_str}"
if params["row_condition"] is not None:
(
conditional_template_str,
conditional_params,
) = parse_row_condition_string_pandas_engine(
params["row_condition"])
template_str = f"{conditional_template_str}, then {template_str}"
params.update(conditional_params)
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": params,
"styling": styling,
},
})
]
def render(cls, index_links_dict):
sections = []
cta_object = index_links_dict.pop("cta_object", None)
try:
content_blocks = []
# site name header
site_name_header_block = RenderedHeaderContent(
**{
"content_block_type":
"header",
"header":
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": "$title_prefix | $site_name",
"params": {
"site_name": index_links_dict.get(
"site_name"),
"title_prefix": "Data Docs",
},
"styling": {
"params": {
"title_prefix": {
"tag": "strong"
}
}
},
},
}),
"styling": {
"classes": ["col-12", "ge-index-page-site-name-title"],
"header": {
"classes": ["alert", "alert-secondary"]
},
},
})
content_blocks.append(site_name_header_block)
tabs = []
if index_links_dict.get("validations_links"):
tabs.append({
"tab_name":
"Validation Results",
"tab_content":
cls._generate_validation_results_link_table(
index_links_dict),
})
if index_links_dict.get("profiling_links"):
tabs.append({
"tab_name":
"Profiling Results",
"tab_content":
cls._generate_profiling_results_link_table(
index_links_dict),
})
if index_links_dict.get("expectations_links"):
tabs.append({
"tab_name":
"Expectation Suites",
"tab_content":
cls._generate_expectation_suites_link_table(
index_links_dict),
})
tabs_content_block = RenderedTabsContent(
**{
"tabs": tabs,
"styling": {
"classes": ["col-12", "ge-index-page-tabs-container"],
},
})
content_blocks.append(tabs_content_block)
section = RenderedSectionContent(
**{
"section_name": index_links_dict.get("site_name"),
"content_blocks": content_blocks,
})
sections.append(section)
index_page_document = RenderedDocumentContent(
**{
"renderer_type": "SiteIndexPageRenderer",
"utm_medium": "index-page",
"sections": sections,
})
if cta_object:
index_page_document.cta_footer = CallToActionRenderer.render(
cta_object)
return index_page_document
except Exception as e:
exception_message = f"""\
An unexpected Exception occurred during data docs rendering. Because of this error, certain parts of data docs will \
not be rendered properly and/or may not appear altogether. Please use the trace, included in this message, to \
diagnose and repair the underlying issue. Detailed information follows:
"""
exception_traceback = traceback.format_exc()
exception_message += (
f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
)
logger.error(exception_message, e, exc_info=True)
def _get_unexpected_statement(cls, evr):
success = evr.success
result = evr.result
if evr.exception_info["raised_exception"]:
exception_message_template_str = "\n\n$expectation_type raised an exception:\n$exception_message"
exception_message = RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": exception_message_template_str,
"params": {
"expectation_type":
evr.expectation_config.expectation_type,
"exception_message":
evr.exception_info["exception_message"]
},
"tag": "strong",
"styling": {
"classes": ["text-danger"],
"params": {
"exception_message": {
"tag": "code"
},
"expectation_type": {
"classes":
["badge", "badge-danger", "mb-2"]
}
}
}
},
})
exception_traceback_collapse = CollapseContent(
**{
"collapse_toggle_link":
"Show exception traceback...",
"collapse": [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template":
evr.exception_info["exception_traceback"],
"tag":
"code"
}
})
]
})
return [exception_message, exception_traceback_collapse]
if success or not result.get("unexpected_count"):
return []
else:
unexpected_count = num_to_str(result["unexpected_count"],
use_locale=True,
precision=20)
unexpected_percent = num_to_str(result["unexpected_percent"],
precision=4) + "%"
element_count = num_to_str(result["element_count"],
use_locale=True,
precision=20)
template_str = "\n\n$unexpected_count unexpected values found. " \
"$unexpected_percent of $element_count total rows."
return [
RenderedStringTemplateContent(
**{
"content_block_type": "string_template",
"string_template": {
"template": template_str,
"params": {
"unexpected_count": unexpected_count,
"unexpected_percent": unexpected_percent,
"element_count": element_count
},
"tag": "strong",
"styling": {
"classes": ["text-danger"]
}
}
})
]