def render_generic(summary):
template_variables = {} # render_common(summary)
info = VariableInfo(
anchor_id=summary["varid"],
warnings=summary["warnings"],
var_type="Unsupported",
var_name=summary["varname"],
)
table = Table(
[
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
return {
"top": Sequence([info, table, HTML("")], sequence_type="grid"),
"bottom": None,
}
def render_count(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
)
table1 = Table(
[
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
]
)
table2 = Table(
[
{"name": "Mean", "value": summary["mean"], "fmt": "fmt", "alert": False},
{"name": "Minimum", "value": summary["min"], "fmt": "fmt", "alert": False},
{"name": "Maximum", "value": summary["max"], "fmt": "fmt", "alert": False},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary, summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Sequence(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = {
"name": "Quantile statistics",
"items": [
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th percentile",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "median",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th percentile",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Interquartile range",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name": "Descriptive statistics",
"items": [
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "Kurtosis", "value": summary["kurt"], "fmt": "fmt_numeric"},
{"name": "Mean", "value": summary["mean"], "fmt": "fmt_numeric"},
{"name": "MAD", "value": summary["mad"], "fmt": "fmt_numeric"},
{"name": "Skewness", "value": summary["skew"], "fmt": "fmt_numeric"},
{"name": "Square Mean Error (SME)", "value": summary["sme"], "fmt": "fmt_numeric"},
{"name": "Sum", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "Variance", "value": summary["var"], "fmt": "fmt_numeric"},
],
}
seqs = [
Image(
histogram(summary["histogram_data"], summary, summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={summary['histogram_bins']})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
)
evs = Sequence(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
if "histogram_bins_bayesian_blocks" in summary:
histo_dyn = Image(
histogram(
summary["histogram_data"],
summary,
summary["histogram_bins_bayesian_blocks"],
),
image_format=image_format,
alt="Histogram",
caption='<strong>Histogram with variable size bins</strong> (bins={}, <a href="https://ui.adsabs.harvard.edu/abs/2013ApJ...764..167S/abstract" target="_blank">"bayesian blocks"</a> binning strategy used)'.format(
fmt_array(summary["histogram_bins_bayesian_blocks"], threshold=5)
),
name="Dynamic Histogram",
anchor_id="dynamic_histogram",
)
seqs.append(histo_dyn)
template_variables["bottom"] = Sequence(
[
# statistics,
Sequence(
seqs, sequence_type="tabs", name="Histogram(s)", anchor_id="histograms"
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_boolean(summary):
varid = summary["varid"]
n_obs_bool = config["vars"]["bool"]["n_obs"].get(int)
# Prepare variables
template_variables = render_common(summary)
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["n"],
max_number_to_print=n_obs_bool,
)
# Element composition
info = VariableInfo(
anchor_id=summary["varid"],
warnings=summary["warnings"],
var_type="Boolean",
var_name=summary["varname"],
)
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
template_variables["top"] = Sequence([info, table, fqm],
sequence_type="grid")
freqtable = FrequencyTable(
template_variables["freq_table_rows"],
name="Frequency Table",
anchor_id=f"{varid}frequency_table",
)
template_variables["bottom"] = Sequence([freqtable],
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def render_complex(summary):
varid = summary["varid"]
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Complex number (ℂ)",
summary["warnings"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt"
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent"
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt"
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
])
table2 = Table([
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt"
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt"
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt"
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent"
},
])
placeholder = HTML("")
template_variables["top"] = Sequence([info, table1, table2, placeholder],
sequence_type="grid")
# Bottom
items = [
Image(
scatter_complex(summary["scatter_data"]),
image_format=image_format,
alt="Scatterplot",
caption="Scatterplot in the complex plane",
name="Scatter",
anchor_id=f"{varid}scatter",
)
]
bottom = Sequence(items, sequence_type="tabs", anchor_id=summary["varid"])
template_variables["bottom"] = bottom
return template_variables
def render_date(summary):
varid = summary["varid"]
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(summary["varid"], summary["varname"], "Date",
summary["warnings"])
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
])
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Sequence([info, table1, table2, mini_histo],
sequence_type="grid")
# Bottom
bottom = Sequence(
[
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption="Histogram",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_categorical(summary):
varid = summary["varid"]
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_common(summary)
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = VariableInfo(summary["varid"], summary["varname"], "Categorical",
summary["warnings"])
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
template_variables["top"] = Sequence([info, table, fqm],
sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}common_values",
)
items.append(frequency_table)
check_compositions = config["vars"]["cat"]["check_composition"].get(bool)
if check_compositions:
n_freq_table_max = config["n_freq_table_max"].get(int)
citems = []
vc = pd.Series(summary["category_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Categories",
anchor_id=f"{varid}category_long_values",
))
vc = pd.Series(summary["script_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Scripts",
anchor_id=f"{varid}script_values",
))
vc = pd.Series(summary["block_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Blocks",
anchor_id=f"{varid}block_alias_values",
))
characters = Sequence(
citems,
name="Characters",
sequence_type="tabs",
anchor_id=f"{varid}characters",
)
items.append(characters)
template_variables["bottom"] = Sequence(items,
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def render_url(summary):
varid = summary["varid"]
n_freq_table_max = config["n_freq_table_max"].get(int)
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
mini_freq_table_rows = freq_table(freqtable=summary["value_counts"],
n=summary["n"],
max_number_to_print=n_obs_cat)
template_variables = render_common(summary)
keys = ["scheme", "netloc", "path", "query", "fragment"]
for url_part in keys:
template_variables[f"freqtable_{url_part}"] = freq_table(
freqtable=summary[f"{url_part}_counts"],
n=summary["n"],
max_number_to_print=n_freq_table_max,
)
full_frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Full",
anchor_id=f"{varid}full_frequency",
)
scheme_frequency_table = FrequencyTable(
template_variables["freqtable_scheme"],
name="Scheme",
anchor_id=f"{varid}scheme_frequency",
)
netloc_frequency_table = FrequencyTable(
template_variables["freqtable_netloc"],
name="Netloc",
anchor_id=f"{varid}netloc_frequency",
)
path_frequency_table = FrequencyTable(
template_variables["freqtable_path"],
name="Path",
anchor_id=f"{varid}path_frequency",
)
query_frequency_table = FrequencyTable(
template_variables["freqtable_query"],
name="Query",
anchor_id=f"{varid}query_frequency",
)
fragment_frequency_table = FrequencyTable(
template_variables["freqtable_fragment"],
name="Fragment",
anchor_id=f"{varid}fragment_frequency",
)
items = [
full_frequency_table,
scheme_frequency_table,
netloc_frequency_table,
path_frequency_table,
query_frequency_table,
fragment_frequency_table,
]
template_variables["bottom"] = Sequence(items,
sequence_type="tabs",
name="url stats",
anchor_id=f"{varid}urlstats")
# Element composition
info = VariableInfo(summary["varid"], summary["varname"], "URL",
summary["warnings"])
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
template_variables["top"] = Sequence([info, table, fqm],
sequence_type="grid")
return template_variables