def render_categorical_frequency(config: Settings, summary: dict,
varid: str) -> Tuple[Renderable, Renderable]:
frequency_table = Table(
[
{
"name": "Unique",
"value":
f"{summary['n_unique']} {help('The number of unique values (all values that occur exactly once in the dataset).')}",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": fmt_percent(summary["p_unique"]),
"alert": "p_unique" in summary["warn_fields"],
},
],
name="Unique",
anchor_id=f"{varid}_unique_stats",
)
frequencies = Image(
histogram(config, *summary["histogram_frequencies"]),
image_format=config.plot.image_format,
alt="frequencies histogram",
name="Frequencies histogram",
caption="Frequencies of value counts",
anchor_id=f"{varid}frequencies",
)
return frequency_table, frequencies
def render_categorical_frequency(summary, varid, image_format):
frequency_table = Table(
[
{
"name": "Unique",
"value": f"{summary['n_unique']} {help('The number of unique values (all values that occur exactly once in the dataset).')}",
"fmt": "raw",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
],
name="Unique",
anchor_id=f"{varid}_unique_stats",
)
frequencies = Image(
histogram(*summary["histogram_frequencies"]),
image_format=image_format,
alt="frequencies histogram",
name="Frequencies histogram",
caption="Frequencies of value counts",
anchor_id=f"{varid}frequencies",
)
return frequency_table, frequencies
def render_file(summary):
varid = summary["varid"]
template_variables = render_path(summary)
# Top
template_variables["top"].content["items"][0].content["var_type"] = "File"
n_freq_table_max = config["n_freq_table_max"].get(int)
image_format = config["plot"]["image_format"].get(str)
file_tabs = []
if "file_size" in summary:
file_tabs.append(
Image(
histogram(*summary["histogram_file_size"]),
image_format=image_format,
alt="Size",
caption=
f"<strong>Histogram with fixed size bins of file sizes (in bytes)</strong> (bins={len(summary['histogram_file_size'][1]) - 1})",
name="File size",
anchor_id=f"{varid}file_size_histogram",
))
file_dates = {
"file_created_time": "Created",
"file_accessed_time": "Accessed",
"file_modified_time": "Modified",
}
for file_date_id, description in file_dates.items():
if file_date_id in summary:
file_tabs.append(
FrequencyTable(
freq_table(
freqtable=summary[file_date_id].value_counts(),
n=summary["n"],
max_number_to_print=n_freq_table_max,
),
name=description,
anchor_id=f"{varid}{file_date_id}",
redact=False,
))
file_tab = Container(
file_tabs,
name="File",
sequence_type="tabs",
anchor_id=f"{varid}file",
)
template_variables["bottom"].content["items"].append(file_tab)
return template_variables
def render_categorical_length(summary, varid, image_format):
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Median length",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
length = Image(
histogram(*summary["histogram_length"]),
image_format=image_format,
alt="length histogram",
name="Length",
caption="Histogram of lengths of the category",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="Length",
sequence_type="grid",
)
return length_tab
def render_categorical_length(summary, varid, image_format):
length_table = Table(
[
{
"name": "最大长度",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "中位长度",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "平均长度",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最小长度",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="长度",
anchor_id=f"{varid}lengthstats",
)
length = Image(
histogram(*summary["histogram_length"]),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="长度",
sequence_type="grid",
)
return length_tab
def render_categorical_length(config: Settings, summary: dict,
varid: str) -> Tuple[Renderable, Renderable]:
length_table = Table(
[
{
"name": "Max length",
"value": fmt_number(summary["max_length"]),
"alert": False,
},
{
"name": "Median length",
"value": fmt_number(summary["median_length"]),
"alert": False,
},
{
"name":
"Mean length",
"value":
fmt_numeric(summary["mean_length"],
precision=config.report.precision),
"alert":
False,
},
{
"name": "Min length",
"value": fmt_number(summary["min_length"]),
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
length_histo = Image(
histogram(config, *summary["histogram_length"]),
image_format=config.plot.image_format,
alt="length histogram",
name="Length",
caption="Histogram of lengths of the category",
anchor_id=f"{varid}length",
)
return length_table, length_histo
def render_date(summary):
varid = summary["varid"]
# TODO: render common?
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
])
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
# Bottom
bottom = Container(
[
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption="Histogram",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_variables_section(stats_object: dict) -> str:
"""Render the HTML for each of the variables in the DataFrame.
Args:
stats_object: The statistics for each variable.
Returns:
The rendered HTML, where each row represents a variable.
"""
rows_html = u""
n_obs_unique = config["n_obs_unique"].get(int)
n_obs_bool = config["n_obs_bool"].get(int)
n_extreme_obs = config["n_extreme_obs"].get(int)
n_freq_table_max = config["n_freq_table_max"].get(int)
messages = stats_object["messages"]
# TODO: move to for loop in template
for idx, row in stats_object["variables"].items():
formatted_values = row
formatted_values.update({
"varname": idx,
"varid": hash(idx),
"row_classes": {}
})
# TODO: obtain from messages (ignore)
for m in messages:
if m.column_name == idx:
if m.message_type == MessageType.SKEWED:
formatted_values["row_classes"]["skewness"] = "alert"
elif m.message_type == MessageType.HIGH_CARDINALITY:
# TODO: rename alert to prevent overlap with bootstrap classes
formatted_values["row_classes"]["distinct_count"] = "alert"
elif m.message_type == MessageType.ZEROS:
formatted_values["row_classes"]["zeros"] = "alert"
elif m.message_type == MessageType.MISSING:
formatted_values["row_classes"]["missing"] = "alert"
if row["type"] in {Variable.TYPE_NUM, Variable.TYPE_DATE}:
formatted_values["histogram"] = histogram(row["histogramdata"],
row,
row["histogram_bins"])
formatted_values["mini_histogram"] = mini_histogram(
row["histogramdata"], row, row["histogram_bins"])
if ("histogram_bins_bayesian_blocks" in row
and row["type"] == Variable.TYPE_NUM):
formatted_values["histogram_bayesian_blocks"] = histogram(
row["histogramdata"], row,
row["histogram_bins_bayesian_blocks"])
if row["type"] in {Variable.TYPE_CAT, Variable.TYPE_BOOL}:
# The number of column to use in the display of the frequency table according to the category
mini_freq_table_nb_col = {
Variable.TYPE_CAT: 6,
Variable.TYPE_BOOL: 3
}
formatted_values["minifreqtable"] = freq_table(
stats_object["variables"][idx]["value_counts_without_nan"],
stats_object["table"]["n"],
"mini_freq_table.html",
max_number_to_print=n_obs_bool,
idx=idx,
nb_col=mini_freq_table_nb_col[row["type"]],
)
if row["type"] in {Variable.TYPE_URL}:
keys = ["scheme", "netloc", "path", "query", "fragment"]
for url_part in keys:
formatted_values["freqtable_{}".format(url_part)] = freq_table(
freqtable=stats_object["variables"][idx][
"{}_counts".format(url_part)],
# TODO: n - missing
n=stats_object["table"]["n"],
table_template="freq_table.html",
idx=idx,
max_number_to_print=n_freq_table_max,
)
if row["type"] in {Variable.TYPE_PATH}:
keys = ["name", "parent", "suffix", "stem"]
for path_part in keys:
formatted_values["freqtable_{}".format(
path_part)] = freq_table(
freqtable=stats_object["variables"][idx][
"{}_counts".format(path_part)],
# TODO: n - missing
n=stats_object["table"]["n"],
table_template="freq_table.html",
idx=idx,
max_number_to_print=n_freq_table_max,
)
if row["type"] == Variable.S_TYPE_UNIQUE:
table = stats_object["variables"][idx][
"value_counts_without_nan"].sort_index()
obs = table.index
formatted_values["firstn"] = pd.DataFrame(
list(obs[0:n_obs_unique]),
columns=["First {} values".format(n_obs_unique)],
).to_html(classes="example_values", index=False)
formatted_values["lastn"] = pd.DataFrame(
list(obs[-n_obs_unique:]),
columns=["Last {} values".format(n_obs_unique)],
).to_html(classes="example_values", index=False)
if row["type"] not in {
Variable.S_TYPE_UNSUPPORTED,
Variable.S_TYPE_CORR,
Variable.S_TYPE_CONST,
Variable.S_TYPE_RECODED,
}:
formatted_values["freqtable"] = freq_table(
freqtable=stats_object["variables"][idx]
["value_counts_without_nan"],
n=stats_object["table"]["n"],
table_template="freq_table.html",
idx=idx,
max_number_to_print=n_freq_table_max,
)
formatted_values["n_extreme_obs"] = n_extreme_obs
formatted_values["firstn_expanded"] = extreme_obs_table(
freqtable=stats_object["variables"][idx]
["value_counts_without_nan"],
number_to_print=n_extreme_obs,
n=stats_object["table"]["n"],
ascending=True,
)
formatted_values["lastn_expanded"] = extreme_obs_table(
freqtable=stats_object["variables"][idx]
["value_counts_without_nan"],
number_to_print=n_extreme_obs,
n=stats_object["table"]["n"],
ascending=False,
)
if row["type"] == Variable.TYPE_NUM:
formatted_values["sections"] = {
"statistics": {
"name":
"Statistics",
"content":
templates.template("variables/row_num_statistics.html").
render(values=formatted_values),
},
"histogram": {
"name":
"Histogram",
"content":
templates.template("variables/row_num_histogram.html").
render(values=formatted_values),
},
"frequency_table": {
"name":
"Common values",
"content":
templates.template("variables/row_num_frequency_table.html"
).render(values=formatted_values),
},
"extreme_values": {
"name":
"Extreme values",
"content":
templates.template("variables/row_num_extreme_values.html"
).render(values=formatted_values),
},
}
if row["type"] == Variable.TYPE_CAT:
formatted_values["sections"] = {
"frequency_table": {
"name":
"Common values",
"content":
templates.template("variables/row_cat_frequency_table.html"
).render(values=formatted_values),
}
}
check_compositions = config["vars"]["cat"][
"check_composition"].get(bool)
if check_compositions:
formatted_values["sections"]["composition"] = {
"name":
"Composition",
"content":
templates.template("variables/row_cat_composition.html").
render(values=formatted_values),
}
if row["type"] == Variable.TYPE_URL:
formatted_values["sections"] = {
"full": {
"name": "Full",
"value": formatted_values["freqtable"]
},
"scheme": {
"name": "Scheme",
"value": formatted_values["freqtable_scheme"],
},
"netloc": {
"name": "Netloc",
"value": formatted_values["freqtable_netloc"],
},
"path": {
"name": "Path",
"value": formatted_values["freqtable_path"]
},
"query": {
"name": "Query",
"value": formatted_values["freqtable_query"],
},
"fragment": {
"name": "Fragment",
"value": formatted_values["freqtable_fragment"],
},
}
if row["type"] == Variable.TYPE_PATH:
formatted_values["sections"] = {
"full": {
"name": "Full",
"value": formatted_values["freqtable"]
},
"stem": {
"name": "Stem",
"value": formatted_values["freqtable_stem"]
},
"name": {
"name": "Name",
"value": formatted_values["freqtable_name"]
},
"suffix": {
"name": "Suffix",
"value": formatted_values["freqtable_suffix"],
},
"parent": {
"name": "Parent",
"value": formatted_values["freqtable_parent"],
},
}
rows_html += templates.template("variables/row_{}.html".format(
row["type"].value.lower())).render(values=formatted_values)
return rows_html
def render_path(summary):
varid = summary["varid"]
n_freq_table_max = config["n_freq_table_max"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_categorical(summary)
keys = ["name", "parent", "suffix", "stem"]
for path_part in keys:
template_variables[f"freqtable_{path_part}"] = freq_table(
freqtable=summary[f"{path_part}_counts"],
n=summary["n"],
max_number_to_print=n_freq_table_max,
)
# Top
template_variables["top"].content["items"][0].content["var_type"] = "Path"
# TODO: colspan=2
# template_variables['top'].content['items'][1].content['rows'].append({'name': 'Common prefix', 'value': summary['common_prefix'], 'fmt': 'fmt'})
# { # <td>#}
# { # <div style="white-space: nowrap;overflow: hidden;text-overflow: ellipsis;max-width: 600px;">#}
# { # {{ values['common_prefix'] }}#}
# { # </div>#}
# { # </td>#}
#
# Bottom
full = FrequencyTable(
template_variables["freq_table_rows"],
name="Full",
anchor_id=f"{varid}full_frequency",
)
stem = FrequencyTable(
template_variables["freqtable_stem"],
name="Stem",
anchor_id=f"{varid}stem_frequency",
)
name = FrequencyTable(
template_variables["freqtable_name"],
name="Name",
anchor_id=f"{varid}name_frequency",
)
suffix = FrequencyTable(
template_variables["freqtable_suffix"],
name="Suffix",
anchor_id=f"{varid}suffix_frequency",
)
parent = FrequencyTable(
template_variables["freqtable_parent"],
name="Parent",
anchor_id=f"{varid}parent_frequency",
)
template_variables["bottom"].content["items"].append(full)
template_variables["bottom"].content["items"].append(stem)
template_variables["bottom"].content["items"].append(name)
template_variables["bottom"].content["items"].append(suffix)
template_variables["bottom"].content["items"].append(parent)
if "file_sizes" in summary:
file_size_histogram = Image(
histogram(summary["file_sizes"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="File size",
caption=
f"<strong>Histogram with fixed size bins of file sizes (in bytes)</strong> (bins={summary['histogram_bins']})",
name="File size",
anchor_id=f"{varid}file_size_histogram",
)
# TODO: in SequeencyItem
template_variables["bottom"].content["items"].append(
file_size_histogram)
return template_variables
def render_categorical(summary):
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = Overview(summary["varid"], summary["varname"], "Categorical",
summary["warnings"])
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"class": "alert" if "n_unique" in summary["warn_fields"] else "",
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"class": "alert" if "p_unique" in summary["warn_fields"] else "",
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"class": "alert" if "n_missing" in summary["warn_fields"] else "",
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"class": "alert" if "p_missing" in summary["warn_fields"] else "",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Sequence([info, table, fqm],
sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id="{varid}common_values".format(varid=summary["varid"]),
)
items.append(frequency_table)
check_compositions = config["vars"]["cat"]["check_composition"].get(bool)
if check_compositions:
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
},
],
name="Length",
anchor_id="{varid}lengthstats".format(varid=summary["varid"]),
)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id="{varid}length".format(varid=summary["varid"]),
)
tbl = Sequence(
[length, length_table],
anchor_id="{varid}tbl".format(varid=summary["varid"]),
name="Length",
sequence_type="grid",
)
items.append(tbl)
n_freq_table_max = config["n_freq_table_max"].get(int)
citems = []
vc = pd.Series(summary["category_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Categories",
anchor_id="{varid}category_long_values".format(
varid=summary["varid"]),
))
vc = pd.Series(summary["script_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Scripts",
anchor_id="{varid}script_values".format(
varid=summary["varid"]),
))
vc = pd.Series(summary["block_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Blocks",
anchor_id="{varid}block_alias_values".format(
varid=summary["varid"]),
))
characters = Sequence(
citems,
name="Characters",
sequence_type="tabs",
anchor_id="{varid}characters".format(varid=summary["varid"]),
)
items.append(characters)
template_variables["bottom"] = Sequence(
items,
sequence_type="tabs",
anchor_id="{varid}bottom".format(varid=summary["varid"]),
)
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "唯一值比例 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "缺失值比例(%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "无穷值",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "无穷值比例 (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
]
)
table2 = Table(
[
{
"name": "均数",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最小值",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最大值",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "零值",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "零值比例 (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = Table(
[
{"name": "最小值", "value": summary["min"], "fmt": "fmt_numeric"},
{"name": "5百分位", "value": summary["5%"], "fmt": "fmt_numeric"},
{"name": "25百分位", "value": summary["25%"], "fmt": "fmt_numeric"},
{"name": "中位", "value": summary["50%"], "fmt": "fmt_numeric"},
{"name": "75百分位", "value": summary["75%"], "fmt": "fmt_numeric"},
{"name": "95-百分位", "value": summary["95%"], "fmt": "fmt_numeric"},
{"name": "最大值", "value": summary["max"], "fmt": "fmt_numeric"},
{"name": "极差", "value": summary["range"], "fmt": "fmt_numeric"},
{
"name": "四分位距 (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="定性统计",
)
if summary["monotonic_increase_strict"]:
monotocity = "严格递增"
elif summary["monotonic_decrease_strict"]:
monotocity = "严格递减"
elif summary["monotonic_increase"]:
monotocity = "递增"
elif summary["monotonic_decrease"]:
monotocity = "递减"
else:
monotocity = "非单调"
descriptive_statistics = Table(
[
{
"name": "标准差",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "变异系数 (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "峰度", "value": summary["kurtosis"], "fmt": "fmt_numeric"},
{"name": "均数", "value": summary["mean"], "fmt": "fmt_numeric"},
{
"name": "中位绝对偏差 (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "偏度",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class": "alert" if "skewness" in summary["warn_fields"] else "",
},
{"name": "总和", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "方差", "value": summary["variance"], "fmt": "fmt_numeric"},
{"name": "单调性", "value": monotocity, "fmt": "fmt"},
],
name="描述性统计",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="统计",
sequence_type="grid",
)
hist = Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>固定大小的直方图</strong> (bins={len(summary['histogram'][1]) - 1})",
name="直方图",
anchor_id=f"{varid}histogram",
)
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="常见值",
anchor_id=f"{varid}common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="最小10个",
anchor_id=f"{varid}firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="最大10个",
anchor_id=f"{varid}lastn",
redact=False,
),
],
sequence_type="tabs",
name="极值",
anchor_id=f"{varid}extreme_values",
)
template_variables["bottom"] = Container(
[statistics, hist, fq, evs], sequence_type="tabs", anchor_id=f"{varid}bottom",
)
return template_variables
def render_date(config: Settings, summary: Dict[str, Any]) -> Dict[str, Any]:
varid = summary["varid"]
template_variables = {}
image_format = config.plot.image_format
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": False,
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": False,
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": False,
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": False,
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
]
)
table2 = Table(
[
{"name": "Minimum", "value": fmt(summary["min"]), "alert": False},
{"name": "Maximum", "value": fmt(summary["max"]), "alert": False},
]
)
mini_histo = Image(
mini_histogram(
config, summary["histogram"][0], summary["histogram"][1], date=True
),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
# Bottom
bottom = Container(
[
Image(
histogram(
config, summary["histogram"][0], summary["histogram"][1], date=True
),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_date(summary):
varid = summary["varid"]
# TODO: render common?
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "唯一值比例 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "缺失值比例(%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
table2 = Table(
[
{"name": "最小", "value": summary["min"], "fmt": "fmt", "alert": False},
{"name": "最大", "value": summary["max"], "fmt": "fmt", "alert": False},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"], date=True),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
# Bottom
bottom = Container(
[
Image(
histogram(*summary["histogram"], date=True),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_count(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "Distinct",
"value": summary["n_distinct"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
]
)
table2 = Table(
[
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = {
"name": "Quantile statistics",
"items": [
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th percentile",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "median",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th percentile",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Interquartile range",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name": "Descriptive statistics",
"items": [
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "Kurtosis", "value": summary["kurt"], "fmt": "fmt_numeric"},
{"name": "Mean", "value": summary["mean"], "fmt": "fmt_numeric"},
{"name": "MAD", "value": summary["mad"], "fmt": "fmt_numeric"},
{"name": "Skewness", "value": summary["skew"], "fmt": "fmt_numeric"},
{"name": "Sum", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "Variance", "value": summary["var"], "fmt": "fmt_numeric"},
],
}
# TODO: Make sections data structure
# statistics = ItemRenderer(
# 'statistics',
# 'Statistics',
# 'table',
# [
# quantile_statistics,
# descriptive_statistics
# ]
# )
seqs = [
Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
template_variables["bottom"] = Container(
[
# statistics,
Container(
seqs, sequence_type="tabs", name="Histogram(s)", anchor_id="histograms"
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_categorical(summary):
varid = summary["varid"]
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = VariableInfo(
summary["varid"],
summary["varname"],
"Categorical",
summary["warnings"],
summary["description"],
)
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Container([info, table, fqm],
sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}common_values",
)
items.append(frequency_table)
check_length = config["vars"]["cat"]["length"].get(bool)
if check_length:
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Median length",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="Length",
sequence_type="grid",
)
items.append(length_tab)
check_unicode = config["vars"]["cat"]["unicode"].get(bool)
if check_unicode:
n_freq_table_max = config["n_freq_table_max"].get(int)
category_items = [
FrequencyTable(
freq_table(
freqtable=summary["category_alias_counts"],
n=summary["category_alias_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring categories",
anchor_id=f"{varid}category_long_values",
)
]
for category_alias_name, category_alias_counts in summary[
"category_alias_char_counts"].items():
category_alias_name = category_alias_name.replace("_", " ")
category_items.append(
FrequencyTable(
freq_table(
freqtable=category_alias_counts,
n=category_alias_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {category_alias_name} characters",
anchor_id=
f"{varid}category_alias_values_{category_alias_name}",
))
script_items = [
FrequencyTable(
freq_table(
freqtable=summary["script_counts"],
n=summary["script_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring scripts",
anchor_id=f"{varid}script_values",
),
]
for script_name, script_counts in summary["script_char_counts"].items(
):
script_items.append(
FrequencyTable(
freq_table(
freqtable=script_counts,
n=script_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {script_name} characters",
anchor_id=f"{varid}script_values_{script_name}",
))
block_items = [
FrequencyTable(
freq_table(
freqtable=summary["block_alias_counts"],
n=summary["block_alias_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring blocks",
anchor_id=f"{varid}block_alias_values",
)
]
for block_name, block_counts in summary[
"block_alias_char_counts"].items():
block_items.append(
FrequencyTable(
freq_table(
freqtable=block_counts,
n=block_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {block_name} characters",
anchor_id=f"{varid}block_alias_values_{block_name}",
))
citems = [
Container(
[
Table(
[
{
"name": "Unique unicode characters",
"value": summary["n_characters"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode categories (<a target="_blank" href="https://en.wikipedia.org/wiki/Unicode_character_property#General_Category">?</a>)',
"value": summary["n_category"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode scripts (<a target="_blank" href="https://en.wikipedia.org/wiki/Script_(Unicode)#List_of_scripts_in_Unicode">?</a>)',
"value": summary["n_scripts"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode blocks (<a target="_blank" href="https://en.wikipedia.org/wiki/Unicode_block">?</a>)',
"value": summary["n_block_alias"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Overview of Unicode Properties",
caption=
"The Unicode Standard assigns character properties to each code point, which can be used to analyse textual variables. ",
),
],
anchor_id=f"{varid}character_overview",
name="Overview",
sequence_type="list",
),
Container(
[
FrequencyTable(
freq_table(
freqtable=summary["character_counts"],
n=summary["character_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring characters",
anchor_id=f"{varid}character_frequency",
),
],
name="Characters",
anchor_id=f"{varid}characters",
sequence_type="named_list",
),
Container(
category_items,
name="Categories",
anchor_id=f"{varid}categories",
sequence_type="named_list",
),
Container(
script_items,
name="Scripts",
anchor_id=f"{varid}scripts",
sequence_type="named_list",
),
Container(
block_items,
name="Blocks",
anchor_id=f"{varid}blocks",
sequence_type="named_list",
),
]
characters = Container(
citems,
name="Unicode",
sequence_type="tabs",
anchor_id=f"{varid}unicode",
)
items.append(characters)
template_variables["bottom"] = Container(items,
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct",
"value": summary["n_distinct"],
"fmt": "fmt",
"alert": "n_distinct" in summary["warn_fields"],
},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
"alert": "p_distinct" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Infinite",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "Infinite (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "Negative",
"value": summary["n_negative"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Negative (%)",
"value": summary["p_negative"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = Table(
[
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric"
},
{
"name": "5-th percentile",
"value": summary["5%"],
"fmt": "fmt_numeric"
},
{
"name": "Q1",
"value": summary["25%"],
"fmt": "fmt_numeric"
},
{
"name": "median",
"value": summary["50%"],
"fmt": "fmt_numeric"
},
{
"name": "Q3",
"value": summary["75%"],
"fmt": "fmt_numeric"
},
{
"name": "95-th percentile",
"value": summary["95%"],
"fmt": "fmt_numeric"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric"
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric"
},
{
"name": "Interquartile range (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="Quantile statistics",
)
descriptive_statistics = Table(
[
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurtosis"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "Median Absolute Deviation (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "Skewness",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class":
"alert" if "skewness" in summary["warn_fields"] else "",
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["variance"],
"fmt": "fmt_numeric"
},
{
"name": "Monotonicity",
"value": summary["monotonic"],
"fmt": "fmt_monotonic",
},
],
name="Descriptive statistics",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="Statistics",
sequence_type="grid",
)
hist = Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id=f"{varid}common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id=f"{varid}firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id=f"{varid}lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id=f"{varid}extreme_values",
)
template_variables["bottom"] = Container(
[statistics, hist, fq, evs],
sequence_type="tabs",
anchor_id=f"{varid}bottom",
)
return template_variables
def render_count(summary):
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
])
table2 = Table([
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt",
"alert": False
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
# TODO: replace with SmallImage...
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Sequence([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = {
"name":
"Quantile statistics",
"items": [
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th percentile",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "median",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th percentile",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Interquartile range",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name":
"Descriptive statistics",
"items": [
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurt"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "MAD",
"value": summary["mad"],
"fmt": "fmt_numeric"
},
{
"name": "Skewness",
"value": summary["skew"],
"fmt": "fmt_numeric"
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["var"],
"fmt": "fmt_numeric"
},
],
}
# TODO: Make sections data structure
# statistics = ItemRenderer(
# 'statistics',
# 'Statistics',
# 'table',
# [
# quantile_statistics,
# descriptive_statistics
# ]
# )
seqs = [
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption="<strong>Histogram with fixed size bins</strong> (bins={})"
.format(summary["histogram_bins"]),
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
)
evs = Sequence(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
if "histogram_bins_bayesian_blocks" in summary:
histo_dyn = Image(
histogram(
summary["histogram_data"],
summary,
summary["histogram_bins_bayesian_blocks"],
),
image_format=image_format,
alt="Histogram",
caption=
'<strong>Histogram with variable size bins</strong> (bins={}, <a href="https://ui.adsabs.harvard.edu/abs/2013ApJ...764..167S/abstract" target="_blank">"bayesian blocks"</a> binning strategy used)'
.format(
fmt_array(summary["histogram_bins_bayesian_blocks"],
threshold=5)),
name="Dynamic Histogram",
anchor_id="dynamic_histogram",
)
seqs.append(histo_dyn)
template_variables["bottom"] = Sequence(
[
# statistics,
Sequence(seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id="histograms"),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_count(config: Settings, summary: dict) -> dict:
template_variables = render_common(config, summary)
image_format = config.plot.image_format
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": False,
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": False,
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": False,
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": False,
},
])
table2 = Table([
{
"name":
"Mean",
"value":
fmt_numeric(summary["mean"], precision=config.report.precision),
"alert":
False,
},
{
"name": "Minimum",
"value": fmt_numeric(summary["min"],
precision=config.report.precision),
"alert": False,
},
{
"name": "Maximum",
"value": fmt_numeric(summary["max"],
precision=config.report.precision),
"alert": False,
},
{
"name": "Zeros",
"value": fmt(summary["n_zeros"]),
"alert": False,
},
{
"name": "Zeros (%)",
"value": fmt_percent(summary["p_zeros"]),
"alert": False,
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
])
mini_histo = Image(
mini_histogram(config, *summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
seqs = [
Image(
histogram(config, *summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
template_variables["bottom"] = Container(
[
Container(seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id="histograms"),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Infinite",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "Infinite (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
])
table2 = Table([
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt",
"alert": False
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
histogram_bins = 10
# TODO: replace with SmallImage...
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary, histogram_bins),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = Table(
[
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric"
},
{
"name": "5-th percentile",
"value": summary["5%"],
"fmt": "fmt_numeric"
},
{
"name": "Q1",
"value": summary["25%"],
"fmt": "fmt_numeric"
},
{
"name": "median",
"value": summary["50%"],
"fmt": "fmt_numeric"
},
{
"name": "Q3",
"value": summary["75%"],
"fmt": "fmt_numeric"
},
{
"name": "95-th percentile",
"value": summary["95%"],
"fmt": "fmt_numeric"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric"
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric"
},
{
"name": "Interquartile range (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="Quantile statistics",
)
descriptive_statistics = Table(
[
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurtosis"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "Median Absolute Deviation (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "Skewness",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class":
"alert" if "skewness" in summary["warn_fields"] else "",
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["variance"],
"fmt": "fmt_numeric"
},
],
name="Descriptive statistics",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="Statistics",
sequence_type="grid",
)
seqs = [
Image(
histogram(summary["histogram_data"], summary, histogram_bins),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={histogram_bins})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id=f"{varid}common_values",
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id=f"{varid}firstn",
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id=f"{varid}lastn",
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id=f"{varid}extreme_values",
)
if "histogram_bins_bayesian_blocks" in summary:
histo_dyn = Image(
histogram(
summary["histogram_data"],
summary,
summary["histogram_bins_bayesian_blocks"],
),
image_format=image_format,
alt="Histogram",
caption=
'<strong>Histogram with variable size bins</strong> (bins={}, <a href="https://ui.adsabs.harvard.edu/abs/2013ApJ...764..167S/abstract" target="_blank">"bayesian blocks"</a> binning strategy used)'
.format(
fmt_array(summary["histogram_bins_bayesian_blocks"],
threshold=5)),
name="Dynamic Histogram",
anchor_id=f"{varid}dynamic_histogram",
)
seqs.append(histo_dyn)
template_variables["bottom"] = Container(
[
statistics,
Container(
seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id=f"{varid}histograms",
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=f"{varid}bottom",
)
return template_variables
def render_date(summary):
# TODO: render common?
template_variables = {}
# Top
info = Overview(summary["varid"], summary["varname"], "Date", [])
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt"
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent"
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt"
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt"
},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
])
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
"Mini histogram",
)
template_variables["top"] = Sequence([info, table1, table2, mini_histo],
sequence_type="grid")
# Bottom
bottom = Sequence(
[
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
alt="Histogram",
caption="Histogram",
name="Histogram",
anchor_id="{varid}histogram".format(varid=summary["varid"]),
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_categorical(summary):
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = Overview(
summary["varid"], summary["varname"], "Categorical", summary["warnings"]
)
table = Table(
[
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"class": "alert" if "n_unique" in summary["warn_fields"] else "",
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"class": "alert" if "p_unique" in summary["warn_fields"] else "",
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"class": "alert" if "n_missing" in summary["warn_fields"] else "",
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"class": "alert" if "p_missing" in summary["warn_fields"] else "",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
]
)
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Sequence([info, table, fqm], sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
# 'frequency_table',
template_variables["freq_table_rows"],
name="Common Values",
anchor_id="{varid}common_values".format(varid=summary["varid"]),
)
items.append(frequency_table)
check_compositions = config["vars"]["cat"]["check_composition"].get(bool)
if check_compositions:
composition = Table(
[
{
"name": "Contains chars",
"value": summary["composition"]["chars"],
"fmt": "fmt",
},
{
"name": "Contains digits",
"value": summary["composition"]["digits"],
"fmt": "fmt",
},
{
"name": "Contains whitespace",
"value": summary["composition"]["spaces"],
"fmt": "fmt",
},
{
"name": "Contains non-words",
"value": summary["composition"]["non-words"],
"fmt": "fmt",
},
],
name="Composition",
anchor_id="{varid}composition".format(varid=summary["varid"]),
)
length = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
},
],
name="Length",
anchor_id="{varid}lengthstats".format(varid=summary["varid"]),
)
tbl = Sequence(
[composition, length],
anchor_id="{varid}tbl".format(varid=summary["varid"]),
name="Composition",
sequence_type="grid",
)
items.append(tbl)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
alt="Scatter",
name="Length",
anchor_id="{varid}length".format(varid=summary["varid"]),
)
items.append(length)
template_variables["bottom"] = Sequence(
items,
sequence_type="tabs",
anchor_id="{varid}bottom".format(varid=summary["varid"]),
)
return template_variables
def render_count(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "唯一值 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "缺失值比例 (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
]
)
table2 = Table(
[
{
"name": "均数",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最小值",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最大值",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "零值",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "零值 (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = {
"name": "定性分析",
"items": [
{
"name": "最小值",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th 百分位",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "中位数",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th 百分位",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最大值",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "区间",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "四分位距",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name": "描述性统计",
"items": [
{
"name": "标准差",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "变异系数",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "峰度", "value": summary["kurt"], "fmt": "fmt_numeric"},
{"name": "均数", "value": summary["mean"], "fmt": "fmt_numeric"},
{"name": "MAD", "value": summary["mad"], "fmt": "fmt_numeric"},
{"name": "偏度", "value": summary["skew"], "fmt": "fmt_numeric"},
{"name": "积", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "方差", "value": summary["var"], "fmt": "fmt_numeric"},
],
}
# TODO: Make sections data structure
# statistics = ItemRenderer(
# 'statistics',
# 'Statistics',
# 'table',
# [
# quantile_statistics,
# descriptive_statistics
# ]
# )
seqs = [
Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
redact=False,
),
],
sequence_type="tabs",
name="极值",
anchor_id="extreme_values",
)
template_variables["bottom"] = Container(
[
# statistics,
Container(
seqs, sequence_type="tabs", name="直方图", anchor_id="histograms"
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
