def get_missing_items(summary) -> list:
image_format = config["plot"]["image_format"].get(str)
items = []
for key, item in summary["missing"].items():
items.append(
# TODO: Add informative caption
Image(
item["matrix"],
image_format=image_format,
alt=item["name"],
name=item["name"],
anchor_id=key,
))
return items
def render_categorical_length(summary, varid, image_format):
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Median length",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
length = Image(
histogram(*summary["histogram_length"]),
image_format=image_format,
alt="length histogram",
name="Length",
caption="Histogram of lengths of the category",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="Length",
sequence_type="grid",
)
return length_tab
def render_categorical_length(summary, varid, image_format):
length_table = Table(
[
{
"name": "最大长度",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "中位长度",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "平均长度",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最小长度",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="长度",
anchor_id=f"{varid}lengthstats",
)
length = Image(
histogram(*summary["histogram_length"]),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="长度",
sequence_type="grid",
)
return length_tab
def render_categorical_length(config: Settings, summary: dict,
varid: str) -> Tuple[Renderable, Renderable]:
length_table = Table(
[
{
"name": "Max length",
"value": fmt_number(summary["max_length"]),
"alert": False,
},
{
"name": "Median length",
"value": fmt_number(summary["median_length"]),
"alert": False,
},
{
"name":
"Mean length",
"value":
fmt_numeric(summary["mean_length"],
precision=config.report.precision),
"alert":
False,
},
{
"name": "Min length",
"value": fmt_number(summary["min_length"]),
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
length_histo = Image(
histogram(config, *summary["histogram_length"]),
image_format=config.plot.image_format,
alt="length histogram",
name="Length",
caption="Histogram of lengths of the category",
anchor_id=f"{varid}length",
)
return length_table, length_histo
def get_scatter_matrix(scatter_matrix: dict) -> list:
"""Returns the interaction components for the report
Args:
scatter_matrix: a nested dict containing the scatter plots
Returns:
A list of components for the interaction section of the report
"""
image_format = config["plot"]["image_format"].get(str)
titems = []
alphanum = re.compile("[^a-zA-Z\s]")
def clean_name(name):
return alphanum.sub("", name).replace(" ", "_")
for x_col, y_cols in scatter_matrix.items():
items = []
for y_col, splot in y_cols.items():
items.append(
Image(
splot,
image_format=image_format,
alt=f"{x_col} x {y_col}",
anchor_id=f"interactions_{clean_name(x_col)}_{clean_name(y_col)}",
name=y_col,
)
)
titems.append(
Container(
items,
sequence_type="tabs" if len(items) <= 10 else "select",
name=x_col,
nested=len(scatter_matrix) > 10,
anchor_id=f"interactions_{clean_name(x_col)}",
)
)
return titems
def get_missing_items(summary) -> list:
"""Return the missing diagrams
Args:
summary: the dataframe summary
Returns:
A list with the missing diagrams
"""
image_format = config["plot"]["image_format"].get(str)
items = []
for key, item in summary["missing"].items():
items.append(
# TODO: Add informative caption
Image(
item["matrix"],
image_format=image_format,
alt=item["name"],
name=item["name"],
anchor_id=key,
))
return items
def render_categorical_frequency(summary, varid, image_format):
frequency_table = Table(
[
{
"name": "Unique",
"value":
f"{summary['n_unique']} {help('The number of unique values (all values that occur exactly once in the dataset).')}",
"fmt": "raw",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
],
name="Unique",
anchor_id=f"{varid}uniquenessstats",
)
frequencies = Image(
histogram(*summary["histogram_frequencies"]),
image_format=image_format,
alt="frequencies histogram",
name="Frequencies histogram",
caption="Frequencies of value counts",
anchor_id=f"{varid}frequencies",
)
frequency_tab = Container(
[frequencies, frequency_table],
anchor_id=f"{varid}tbl",
name="Overview",
sequence_type="grid",
)
return frequency_tab
def get_scatter_matrix(scatter_matrix):
image_format = config["plot"]["image_format"].get(str)
titems = []
for x_col, y_cols in scatter_matrix.items():
items = []
for y_col, splot in y_cols.items():
items.append(
Image(
splot,
image_format=image_format,
alt=f"{x_col} x {y_col}",
anchor_id=f"interactions_{x_col}_{y_col}",
name=y_col,
))
titems.append(
Container(
items,
sequence_type="tabs",
name=x_col,
anchor_id=f"interactions_{x_col}",
))
return titems
def render_count(summary):
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
])
table2 = Table([
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt",
"alert": False
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
# TODO: replace with SmallImage...
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Sequence([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = {
"name":
"Quantile statistics",
"items": [
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th percentile",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "median",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th percentile",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Interquartile range",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name":
"Descriptive statistics",
"items": [
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurt"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "MAD",
"value": summary["mad"],
"fmt": "fmt_numeric"
},
{
"name": "Skewness",
"value": summary["skew"],
"fmt": "fmt_numeric"
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["var"],
"fmt": "fmt_numeric"
},
],
}
# TODO: Make sections data structure
# statistics = ItemRenderer(
# 'statistics',
# 'Statistics',
# 'table',
# [
# quantile_statistics,
# descriptive_statistics
# ]
# )
seqs = [
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption="<strong>Histogram with fixed size bins</strong> (bins={})"
.format(summary["histogram_bins"]),
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
)
evs = Sequence(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
if "histogram_bins_bayesian_blocks" in summary:
histo_dyn = Image(
histogram(
summary["histogram_data"],
summary,
summary["histogram_bins_bayesian_blocks"],
),
image_format=image_format,
alt="Histogram",
caption=
'<strong>Histogram with variable size bins</strong> (bins={}, <a href="https://ui.adsabs.harvard.edu/abs/2013ApJ...764..167S/abstract" target="_blank">"bayesian blocks"</a> binning strategy used)'
.format(
fmt_array(summary["histogram_bins_bayesian_blocks"],
threshold=5)),
name="Dynamic Histogram",
anchor_id="dynamic_histogram",
)
seqs.append(histo_dyn)
template_variables["bottom"] = Sequence(
[
# statistics,
Sequence(seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id="histograms"),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct",
"value": summary["n_distinct"],
"fmt": "fmt",
"alert": "n_distinct" in summary["warn_fields"],
},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
"alert": "p_distinct" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Infinite",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "Infinite (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "Negative",
"value": summary["n_negative"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Negative (%)",
"value": summary["p_negative"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = Table(
[
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric"
},
{
"name": "5-th percentile",
"value": summary["5%"],
"fmt": "fmt_numeric"
},
{
"name": "Q1",
"value": summary["25%"],
"fmt": "fmt_numeric"
},
{
"name": "median",
"value": summary["50%"],
"fmt": "fmt_numeric"
},
{
"name": "Q3",
"value": summary["75%"],
"fmt": "fmt_numeric"
},
{
"name": "95-th percentile",
"value": summary["95%"],
"fmt": "fmt_numeric"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric"
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric"
},
{
"name": "Interquartile range (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="Quantile statistics",
)
descriptive_statistics = Table(
[
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurtosis"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "Median Absolute Deviation (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "Skewness",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class":
"alert" if "skewness" in summary["warn_fields"] else "",
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["variance"],
"fmt": "fmt_numeric"
},
{
"name": "Monotonicity",
"value": summary["monotonic"],
"fmt": "fmt_monotonic",
},
],
name="Descriptive statistics",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="Statistics",
sequence_type="grid",
)
hist = Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id=f"{varid}common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id=f"{varid}firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id=f"{varid}lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id=f"{varid}extreme_values",
)
template_variables["bottom"] = Container(
[statistics, hist, fq, evs],
sequence_type="tabs",
anchor_id=f"{varid}bottom",
)
return template_variables
def render_image(config: Settings, summary: dict) -> dict:
varid = summary["varid"]
n_freq_table_max = config.n_freq_table_max
redact = config.vars.cat.redact
template_variables = render_file(config, summary)
# Top
template_variables["top"].content["items"][0].content["var_type"] = "Image"
# Bottom
image_items = []
"""
Min Width Min Height Min Area
Mean Width Mean Height Mean Height
Median Width Median Height Median Height
Max Width Max Height Max Height
All dimension properties are in pixels.
"""
image_shape_items = [
Container(
[
Table([
{
"name":
"Min width",
"value":
fmt_numeric(summary["min_width"],
precision=config.report.precision),
"alert":
False,
},
{
"name":
"Median width",
"value":
fmt_numeric(
summary["median_width"],
precision=config.report.precision,
),
"alert":
False,
},
{
"name":
"Max width",
"value":
fmt_numeric(summary["max_width"],
precision=config.report.precision),
"alert":
False,
},
]),
Table([
{
"name":
"Min height",
"value":
fmt_numeric(summary["min_height"],
precision=config.report.precision),
"alert":
False,
},
{
"name":
"Median height",
"value":
fmt_numeric(
summary["median_height"],
precision=config.report.precision,
),
"alert":
False,
},
{
"name":
"Max height",
"value":
fmt_numeric(summary["max_height"],
precision=config.report.precision),
"alert":
False,
},
]),
Table([
{
"name":
"Min area",
"value":
fmt_numeric(summary["min_area"],
precision=config.report.precision),
"alert":
False,
},
{
"name":
"Median area",
"value":
fmt_numeric(
summary["median_area"],
precision=config.report.precision,
),
"alert":
False,
},
{
"name":
"Max area",
"value":
fmt_numeric(summary["max_area"],
precision=config.report.precision),
"alert":
False,
},
]),
],
anchor_id=f"{varid}tbl",
name="Overview",
sequence_type="grid",
),
Image(
scatter_series(config, summary["image_dimensions"]),
image_format=config.plot.image_format,
alt="Scatter plot of image sizes",
caption="Scatter plot of image sizes",
name="Scatter plot",
anchor_id=f"{varid}image_dimensions_scatter",
),
FrequencyTable(
freq_table(
freqtable=summary["image_dimensions"].value_counts(),
n=summary["n"],
max_number_to_print=n_freq_table_max,
),
name="Common values",
anchor_id=f"{varid}image_dimensions_frequency",
redact=False,
),
]
image_shape = Container(
image_shape_items,
sequence_type="named_list",
name="Dimensions",
anchor_id=f"{varid}image_dimensions",
)
if "exif_keys_counts" in summary:
items = [
FrequencyTable(
freq_table(
freqtable=pd.Series(summary["exif_keys_counts"]),
n=summary["n"],
max_number_to_print=n_freq_table_max,
),
name="Exif keys",
anchor_id=f"{varid}exif_keys",
redact=redact,
)
]
for key, counts in summary["exif_data"].items():
if key == "exif_keys":
continue
items.append(
FrequencyTable(
freq_table(
freqtable=counts,
n=summary["n"],
max_number_to_print=n_freq_table_max,
),
name=key,
anchor_id=f"{varid}_exif_{key}",
redact=redact,
))
image_items.append(
Container(
items,
anchor_id=f"{varid}exif_data",
name="Exif data",
sequence_type="named_list",
))
image_items.append(image_shape)
image_tab = Container(
image_items,
name="Image",
sequence_type="tabs",
anchor_id=f"{varid}image",
)
template_variables["bottom"].content["items"].append(image_tab)
return template_variables
def render_categorical(config: Settings, summary: dict) -> dict:
varid = summary["varid"]
n_obs_cat = config.vars.cat.n_obs
image_format = config.plot.image_format
words = config.vars.cat.words
characters = config.vars.cat.characters
length = config.vars.cat.length
template_variables = render_common(config, summary)
info = VariableInfo(
summary["varid"],
summary["varname"],
"Categorical",
summary["alerts"],
summary["description"],
)
table = Table([
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": "n_distinct" in summary["alert_fields"],
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": "p_distinct" in summary["alert_fields"],
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": "n_missing" in summary["alert_fields"],
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": "p_missing" in summary["alert_fields"],
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
])
fqm = FrequencyTableSmall(
freq_table(
freqtable=summary["value_counts_without_nan"],
n=summary["count"],
max_number_to_print=n_obs_cat,
),
redact=config.vars.cat.redact,
)
template_variables["top"] = Container([info, table, fqm],
sequence_type="grid")
# ============================================================================================
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}common_values",
redact=config.vars.cat.redact,
)
unique_stats = render_categorical_frequency(config, summary, varid)
overview_items = []
if length:
length_table, length_histo = render_categorical_length(
config, summary, varid)
overview_items.append(length_table)
if characters:
overview_table_char, unitab = render_categorical_unicode(
config, summary, varid)
overview_items.append(overview_table_char)
overview_items.append(unique_stats)
if not config.vars.cat.redact:
rows = ("1st row", "2nd row", "3rd row", "4th row", "5th row")
sample = Table(
[{
"name": name,
"value": fmt(value),
"alert": False,
} for name, value in zip(rows, summary["first_rows"])],
name="Sample",
)
overview_items.append(sample)
string_items: List[Renderable] = [frequency_table]
if length:
string_items.append(length_histo)
max_unique = config.plot.pie.max_unique
if max_unique > 0 and summary["n_distinct"] <= max_unique:
string_items.append(
Image(
pie_plot(
config,
summary["value_counts_without_nan"],
legend_kws={"loc": "upper right"},
),
image_format=image_format,
alt="Pie chart",
name="Pie chart",
anchor_id=f"{varid}pie_chart",
))
bottom_items = [
Container(
overview_items,
name="Overview",
anchor_id=f"{varid}overview",
sequence_type="batch_grid",
batch_size=len(overview_items),
titles=False,
),
Container(
string_items,
name="Categories",
anchor_id=f"{varid}string",
sequence_type="batch_grid",
batch_size=len(string_items),
),
]
if words:
woc = freq_table(
freqtable=summary["word_counts"],
n=summary["word_counts"].sum(),
max_number_to_print=10,
)
fqwo = FrequencyTable(
woc,
name="Common words",
anchor_id=f"{varid}cwo",
redact=config.vars.cat.redact,
)
bottom_items.append(
Container(
[fqwo],
name="Words",
anchor_id=f"{varid}word",
sequence_type="grid",
))
if characters:
bottom_items.append(
Container(
[unitab],
name="Characters",
anchor_id=f"{varid}characters",
sequence_type="grid",
))
# Bottom
template_variables["bottom"] = Container(bottom_items,
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def get_correlation_items(summary) -> Optional[Renderable]:
"""Create the list of correlation items
Args:
summary: dict of correlations
Returns:
List of correlation items to show in the interface.
"""
items: List[Renderable] = []
pearson_description = (
"The Pearson's correlation coefficient (<em>r</em>) is a measure of linear correlation "
"between two variables. It's value lies between -1 and +1, -1 indicating total negative "
"linear correlation, 0 indicating no linear correlation and 1 indicating total positive "
"linear correlation. Furthermore, <em>r</em> is invariant under separate changes in location "
"and scale of the two variables, implying that for a linear function the angle to the "
"x-axis does not affect <em>r</em>.<br /><br />To calculate <em>r</em> for two "
"variables <em>X</em> and <em>Y</em>, one divides the covariance of <em>X</em> and "
"<em>Y</em> by the product of their standard deviations. ")
spearman_description = """The Spearman's rank correlation coefficient (<em>ρ</em>) is a measure of monotonic
correlation between two variables, and is therefore better in catching nonlinear monotonic correlations than
Pearson's <em>r</em>. It's value lies between -1 and +1, -1 indicating total negative monotonic correlation,
0 indicating no monotonic correlation and 1 indicating total positive monotonic correlation.<br /><br />To
calculate <em>ρ</em> for two variables <em>X</em> and <em>Y</em>, one divides the covariance of the rank
variables of <em>X</em> and <em>Y</em> by the product of their standard deviations. """
kendall_description = """Similarly to Spearman's rank correlation coefficient, the Kendall rank correlation
coefficient (<em>τ</em>) measures ordinal association between two variables. It's value lies between -1 and +1,
-1 indicating total negative correlation, 0 indicating no correlation and 1 indicating total positive correlation.
<br /><br />To calculate <em>τ</em> for two variables <em>X</em> and <em>Y</em>, one determines the number of
concordant and discordant pairs of observations. <em>τ</em> is given by the number of concordant pairs minus the
discordant pairs divided by the total number of pairs."""
phi_k_description = """Phik (φk) is a new and practical correlation coefficient that works consistently between categorical, ordinal and interval variables, captures non-linear dependency and reverts to the Pearson correlation coefficient in case
of a bivariate normal input distribution. There is extensive documentation available <a href='https://phik.readthedocs.io/en/latest/index.html'>here</a>."""
cramers_description = """Cramér's V is an association measure for nominal random variables. The coefficient ranges from 0 to 1, with 0 indicating independence and 1 indicating perfect association.
The empirical estimators used for Cramér's V have been proved to be biased, even for large samples.
We use a bias-corrected measure that has been proposed by Bergsma in 2013 that can be found <a href='http://stats.lse.ac.uk/bergsma/pdf/cramerV3.pdf'>here</a>."""
key_to_data = {
"pearson": (-1, "Pearson's r", pearson_description),
"spearman": (-1, "Spearman's ρ", spearman_description),
"kendall": (-1, "Kendall's τ", kendall_description),
"phi_k": (0, "Phik (φk)", phi_k_description),
"cramers": (0, "Cramér's V (φc)", cramers_description),
}
image_format = config["plot"]["image_format"].get(str)
for key, item in summary["correlations"].items():
vmin, name, description = key_to_data[key]
diagram = Image(
plot.correlation_matrix(item, vmin=vmin),
image_format=image_format,
alt=name,
anchor_id=f"{key}_diagram",
name=name,
classes="correlation-diagram",
)
if len(description) > 0:
desc = HTML(
f'<div style="padding:20px" class="text-muted"><h3>{name}</h3>{description}</div>',
anchor_id=f"{key}_html",
classes="correlation-description",
)
tbl = Container([diagram, desc],
anchor_id=key,
name=name,
sequence_type="grid")
items.append(tbl)
else:
items.append(diagram)
corr = Container(
items,
sequence_type="tabs",
name="Correlations Tab",
anchor_id="correlations_tab",
)
if len(items) > 0:
btn = ToggleButton(
"Toggle correlation descriptions",
anchor_id="toggle-correlation-description",
name="Toggle correlation descriptions",
)
return Collapse(name="Correlations",
anchor_id="correlations",
button=btn,
item=corr)
else:
return None
def get_correlation_items(summary) -> Optional[Renderable]:
"""Create the list of correlation items
Args:
summary: dict of correlations
Returns:
List of correlation items to show in the interface.
"""
items = get_items()
pearson_description = (
"The Pearson's correlation coefficient (<em>r</em>) is a measure of linear correlation "
"between two variables. It's value lies between -1 and +1, -1 indicating total negative "
"linear correlation, 0 indicating no linear correlation and 1 indicating total positive "
"linear correlation. Furthermore, <em>r</em> is invariant under separate changes in location "
"and scale of the two variables, implying that for a linear function the angle to the "
"x-axis does not affect <em>r</em>.<br /><br />To calculate <em>r</em> for two "
"variables <em>X</em> and <em>Y</em>, one divides the covariance of <em>X</em> and "
"<em>Y</em> by the product of their standard deviations. ")
spearman_description = """The Spearman's rank correlation coefficient (<em>ρ</em>) is a measure of monotonic
correlation between two variables, and is therefore better in catching nonlinear monotonic correlations than
Pearson's <em>r</em>. It's value lies between -1 and +1, -1 indicating total negative monotonic correlation,
0 indicating no monotonic correlation and 1 indicating total positive monotonic correlation.<br /><br />To
calculate <em>ρ</em> for two variables <em>X</em> and <em>Y</em>, one divides the covariance of the rank
variables of <em>X</em> and <em>Y</em> by the product of their standard deviations. """
kendall_description = """Similarly to Spearman's rank correlation coefficient, the Kendall rank correlation
coefficient (<em>τ</em>) measures ordinal association between two variables. It's value lies between -1 and +1,
-1 indicating total negative correlation, 0 indicating no correlation and 1 indicating total positive correlation.
<br /><br />To calculate <em>τ</em> for two variables <em>X</em> and <em>Y</em>, one determines the number of
concordant and discordant pairs of observations. <em>τ</em> is given by the number of concordant pairs minus the
discordant pairs divided by the total number of pairs."""
key_to_data = {
"pearson": (-1, "Pearson's r", pearson_description),
"spearman": (-1, "Spearman's ρ", spearman_description),
"kendall": (-1, "Kendall's τ", kendall_description),
"phi_k": (0, "Phik (φk)", ""),
"cramers": (0, "Cramér's V (φc)", ""),
"recoded": (0, "Recoded", ""),
}
image_format = config["plot"]["image_format"].get(str)
for key, item in summary["correlations"].items():
vmin, name, description = key_to_data[key]
diagram = Image(
plot.correlation_matrix(item, vmin=vmin),
image_format=image_format,
alt=name,
anchor_id="{key}_diagram".format(key=key),
name=name,
classes="correlation-diagram",
)
if len(description) > 0:
desc = HTML(
'<div style="padding:20px" class="text-muted"><h3>{name}</h3>{description}</div>'
.format(description=description, name=name),
anchor_id="{key}_html".format(key=key),
classes="correlation-description",
)
tbl = Sequence([diagram, desc],
anchor_id=key,
name=name,
sequence_type="grid")
items.append(tbl)
else:
items.append(diagram)
corr = Sequence(
items,
sequence_type="tabs",
name="Correlations Tab",
anchor_id="correlations_tab",
)
if len(items) > 0:
btn = ToggleButton(
"Toggle correlation descriptions",
anchor_id="toggle-correlation-description",
name="Toggle correlation descriptions",
)
return Collapse(name="Correlations",
anchor_id="correlations",
button=btn,
item=corr)
else:
return None
def render_date(summary):
varid = summary["varid"]
# TODO: render common?
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
])
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
# Bottom
bottom = Container(
[
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="Histogram",
caption="Histogram",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_count(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "Distinct",
"value": summary["n_distinct"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
]
)
table2 = Table(
[
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": False,
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = {
"name": "Quantile statistics",
"items": [
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "5-th percentile",
"value": summary["quantile_5"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q1",
"value": summary["quantile_25"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "median",
"value": summary["quantile_50"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Q3",
"value": summary["quantile_75"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "95-th percentile",
"value": summary["quantile_95"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Interquartile range",
"value": summary["iqr"],
"fmt": "fmt_numeric",
"alert": False,
},
],
}
descriptive_statistics = {
"name": "Descriptive statistics",
"items": [
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "Kurtosis", "value": summary["kurt"], "fmt": "fmt_numeric"},
{"name": "Mean", "value": summary["mean"], "fmt": "fmt_numeric"},
{"name": "MAD", "value": summary["mad"], "fmt": "fmt_numeric"},
{"name": "Skewness", "value": summary["skew"], "fmt": "fmt_numeric"},
{"name": "Sum", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "Variance", "value": summary["var"], "fmt": "fmt_numeric"},
],
}
# TODO: Make sections data structure
# statistics = ItemRenderer(
# 'statistics',
# 'Statistics',
# 'table',
# [
# quantile_statistics,
# descriptive_statistics
# ]
# )
seqs = [
Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
template_variables["bottom"] = Container(
[
# statistics,
Container(
seqs, sequence_type="tabs", name="Histogram(s)", anchor_id="histograms"
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_categorical(summary):
varid = summary["varid"]
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = VariableInfo(
summary["varid"],
summary["varname"],
"Categorical",
summary["warnings"],
summary["description"],
)
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Container([info, table, fqm],
sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}common_values",
)
items.append(frequency_table)
check_length = config["vars"]["cat"]["length"].get(bool)
if check_length:
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Median length",
"value": summary["median_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Length",
anchor_id=f"{varid}lengthstats",
)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id=f"{varid}length",
)
length_tab = Container(
[length, length_table],
anchor_id=f"{varid}tbl",
name="Length",
sequence_type="grid",
)
items.append(length_tab)
check_unicode = config["vars"]["cat"]["unicode"].get(bool)
if check_unicode:
n_freq_table_max = config["n_freq_table_max"].get(int)
category_items = [
FrequencyTable(
freq_table(
freqtable=summary["category_alias_counts"],
n=summary["category_alias_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring categories",
anchor_id=f"{varid}category_long_values",
)
]
for category_alias_name, category_alias_counts in summary[
"category_alias_char_counts"].items():
category_alias_name = category_alias_name.replace("_", " ")
category_items.append(
FrequencyTable(
freq_table(
freqtable=category_alias_counts,
n=category_alias_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {category_alias_name} characters",
anchor_id=
f"{varid}category_alias_values_{category_alias_name}",
))
script_items = [
FrequencyTable(
freq_table(
freqtable=summary["script_counts"],
n=summary["script_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring scripts",
anchor_id=f"{varid}script_values",
),
]
for script_name, script_counts in summary["script_char_counts"].items(
):
script_items.append(
FrequencyTable(
freq_table(
freqtable=script_counts,
n=script_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {script_name} characters",
anchor_id=f"{varid}script_values_{script_name}",
))
block_items = [
FrequencyTable(
freq_table(
freqtable=summary["block_alias_counts"],
n=summary["block_alias_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring blocks",
anchor_id=f"{varid}block_alias_values",
)
]
for block_name, block_counts in summary[
"block_alias_char_counts"].items():
block_items.append(
FrequencyTable(
freq_table(
freqtable=block_counts,
n=block_counts.sum(),
max_number_to_print=n_freq_table_max,
),
name=f"Most frequent {block_name} characters",
anchor_id=f"{varid}block_alias_values_{block_name}",
))
citems = [
Container(
[
Table(
[
{
"name": "Unique unicode characters",
"value": summary["n_characters"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode categories (<a target="_blank" href="https://en.wikipedia.org/wiki/Unicode_character_property#General_Category">?</a>)',
"value": summary["n_category"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode scripts (<a target="_blank" href="https://en.wikipedia.org/wiki/Script_(Unicode)#List_of_scripts_in_Unicode">?</a>)',
"value": summary["n_scripts"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name":
'Unique unicode blocks (<a target="_blank" href="https://en.wikipedia.org/wiki/Unicode_block">?</a>)',
"value": summary["n_block_alias"],
"fmt": "fmt_numeric",
"alert": False,
},
],
name="Overview of Unicode Properties",
caption=
"The Unicode Standard assigns character properties to each code point, which can be used to analyse textual variables. ",
),
],
anchor_id=f"{varid}character_overview",
name="Overview",
sequence_type="list",
),
Container(
[
FrequencyTable(
freq_table(
freqtable=summary["character_counts"],
n=summary["character_counts"].sum(),
max_number_to_print=n_freq_table_max,
),
name="Most occurring characters",
anchor_id=f"{varid}character_frequency",
),
],
name="Characters",
anchor_id=f"{varid}characters",
sequence_type="named_list",
),
Container(
category_items,
name="Categories",
anchor_id=f"{varid}categories",
sequence_type="named_list",
),
Container(
script_items,
name="Scripts",
anchor_id=f"{varid}scripts",
sequence_type="named_list",
),
Container(
block_items,
name="Blocks",
anchor_id=f"{varid}blocks",
sequence_type="named_list",
),
]
characters = Container(
citems,
name="Unicode",
sequence_type="tabs",
anchor_id=f"{varid}unicode",
)
items.append(characters)
template_variables["bottom"] = Container(items,
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def render_complex(summary):
varid = summary["varid"]
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Complex number (ℂ)",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{"name": "Distinct", "value": summary["n_distinct"], "fmt": "fmt"},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
},
{"name": "Missing", "value": summary["n_missing"], "fmt": "fmt"},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
]
)
table2 = Table(
[
{"name": "Mean", "value": summary["mean"], "fmt": "fmt_numeric"},
{"name": "Minimum", "value": summary["min"], "fmt": "fmt_numeric"},
{"name": "Maximum", "value": summary["max"], "fmt": "fmt_numeric"},
{"name": "Zeros", "value": summary["n_zeros"], "fmt": "fmt_numeric"},
{"name": "Zeros (%)", "value": summary["p_zeros"], "fmt": "fmt_percent"},
]
)
placeholder = HTML("")
template_variables["top"] = Container(
[info, table1, table2, placeholder], sequence_type="grid"
)
# Bottom
items = [
Image(
scatter_complex(summary["scatter_data"]),
image_format=image_format,
alt="Scatterplot",
caption="Scatterplot in the complex plane",
name="Scatter",
anchor_id=f"{varid}scatter",
)
]
bottom = Container(items, sequence_type="tabs", anchor_id=summary["varid"])
template_variables["bottom"] = bottom
return template_variables
def render_categorical(summary):
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = Overview(
summary["varid"], summary["varname"], "Categorical", summary["warnings"]
)
table = Table(
[
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"class": "alert" if "n_unique" in summary["warn_fields"] else "",
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"class": "alert" if "p_unique" in summary["warn_fields"] else "",
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"class": "alert" if "n_missing" in summary["warn_fields"] else "",
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"class": "alert" if "p_missing" in summary["warn_fields"] else "",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
]
)
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Sequence([info, table, fqm], sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
# 'frequency_table',
template_variables["freq_table_rows"],
name="Common Values",
anchor_id="{varid}common_values".format(varid=summary["varid"]),
)
items.append(frequency_table)
check_compositions = config["vars"]["cat"]["check_composition"].get(bool)
if check_compositions:
composition = Table(
[
{
"name": "Contains chars",
"value": summary["composition"]["chars"],
"fmt": "fmt",
},
{
"name": "Contains digits",
"value": summary["composition"]["digits"],
"fmt": "fmt",
},
{
"name": "Contains whitespace",
"value": summary["composition"]["spaces"],
"fmt": "fmt",
},
{
"name": "Contains non-words",
"value": summary["composition"]["non-words"],
"fmt": "fmt",
},
],
name="Composition",
anchor_id="{varid}composition".format(varid=summary["varid"]),
)
length = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
},
],
name="Length",
anchor_id="{varid}lengthstats".format(varid=summary["varid"]),
)
tbl = Sequence(
[composition, length],
anchor_id="{varid}tbl".format(varid=summary["varid"]),
name="Composition",
sequence_type="grid",
)
items.append(tbl)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
alt="Scatter",
name="Length",
anchor_id="{varid}length".format(varid=summary["varid"]),
)
items.append(length)
template_variables["bottom"] = Sequence(
items,
sequence_type="tabs",
anchor_id="{varid}bottom".format(varid=summary["varid"]),
)
return template_variables
def render_categorical(summary):
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_common(summary)
# TODO: merge with boolean
mini_freq_table_rows = freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
)
# Top
# Element composition
info = Overview(summary["varid"], summary["varname"], "Categorical",
summary["warnings"])
table = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"class": "alert" if "n_unique" in summary["warn_fields"] else "",
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"class": "alert" if "p_unique" in summary["warn_fields"] else "",
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"class": "alert" if "n_missing" in summary["warn_fields"] else "",
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"class": "alert" if "p_missing" in summary["warn_fields"] else "",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
])
fqm = FrequencyTableSmall(mini_freq_table_rows)
# TODO: settings 3,3,6
template_variables["top"] = Sequence([info, table, fqm],
sequence_type="grid")
# Bottom
items = []
frequency_table = FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id="{varid}common_values".format(varid=summary["varid"]),
)
items.append(frequency_table)
check_compositions = config["vars"]["cat"]["check_composition"].get(bool)
if check_compositions:
length_table = Table(
[
{
"name": "Max length",
"value": summary["max_length"],
"fmt": "fmt_numeric",
},
{
"name": "Mean length",
"value": summary["mean_length"],
"fmt": "fmt_numeric",
},
{
"name": "Min length",
"value": summary["min_length"],
"fmt": "fmt_numeric",
},
],
name="Length",
anchor_id="{varid}lengthstats".format(varid=summary["varid"]),
)
histogram_bins = 10
length = Image(
histogram(summary["length"], summary, histogram_bins),
image_format=image_format,
alt="Scatter",
name="Length",
anchor_id="{varid}length".format(varid=summary["varid"]),
)
tbl = Sequence(
[length, length_table],
anchor_id="{varid}tbl".format(varid=summary["varid"]),
name="Length",
sequence_type="grid",
)
items.append(tbl)
n_freq_table_max = config["n_freq_table_max"].get(int)
citems = []
vc = pd.Series(summary["category_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Categories",
anchor_id="{varid}category_long_values".format(
varid=summary["varid"]),
))
vc = pd.Series(summary["script_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Scripts",
anchor_id="{varid}script_values".format(
varid=summary["varid"]),
))
vc = pd.Series(summary["block_alias_values"]).value_counts()
citems.append(
FrequencyTable(
freq_table(freqtable=vc,
n=vc.sum(),
max_number_to_print=n_freq_table_max),
name="Blocks",
anchor_id="{varid}block_alias_values".format(
varid=summary["varid"]),
))
characters = Sequence(
citems,
name="Characters",
sequence_type="tabs",
anchor_id="{varid}characters".format(varid=summary["varid"]),
)
items.append(characters)
template_variables["bottom"] = Sequence(
items,
sequence_type="tabs",
anchor_id="{varid}bottom".format(varid=summary["varid"]),
)
return template_variables
def render_categorical(summary):
varid = summary["varid"]
n_obs_cat = config["vars"]["cat"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
redact = config["vars"]["cat"]["redact"].get(bool)
template_variables = render_common(summary)
info = VariableInfo(
summary["varid"],
summary["varname"],
"分类变量",
summary["warnings"],
summary["description"],
)
table = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "唯一值比例 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "缺失值比例(%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
fqm = FrequencyTableSmall(
freq_table(
freqtable=summary["value_counts"],
n=summary["count"],
max_number_to_print=n_obs_cat,
),
redact=redact,
)
template_variables["top"] = Container([info, table, fqm], sequence_type="grid")
# Bottom
items = [
FrequencyTable(
template_variables["freq_table_rows"],
name="常见值",
anchor_id=f"{varid}common_values",
redact=redact,
)
]
max_unique = config["plot"]["pie"]["max_unique"].get(int)
if max_unique > 0 and summary["n_unique"] <= max_unique:
items.append(
Image(
pie_plot(summary["value_counts"], legend_kws={"loc": "upper right"}),
image_format=image_format,
alt="Chart",
name="图表",
anchor_id=f"{varid}pie_chart",
)
)
check_length = config["vars"]["cat"]["length"].get(bool)
if check_length:
items.append(render_categorical_length(summary, varid, image_format))
check_unicode = config["vars"]["cat"]["unicode"].get(bool)
if check_unicode:
items.append(render_categorical_unicode(summary, varid, redact))
template_variables["bottom"] = Container(
items, sequence_type="tabs", anchor_id=f"{varid}bottom"
)
return template_variables
def render_date(summary):
varid = summary["varid"]
# TODO: render common?
template_variables = {}
image_format = config["plot"]["image_format"].get(str)
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": False,
},
{
"name": "唯一值比例 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": False,
},
{
"name": "缺失值比例(%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": False,
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
table2 = Table(
[
{"name": "最小", "value": summary["min"], "fmt": "fmt", "alert": False},
{"name": "最大", "value": summary["max"], "fmt": "fmt", "alert": False},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"], date=True),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
# Bottom
bottom = Container(
[
Image(
histogram(*summary["histogram"], date=True),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "唯一值计数",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "唯一值比例 (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "缺失值",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "缺失值比例(%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "无穷值",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "无穷值比例 (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
]
)
table2 = Table(
[
{
"name": "均数",
"value": summary["mean"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最小值",
"value": summary["min"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "最大值",
"value": summary["max"],
"fmt": "fmt_numeric",
"alert": False,
},
{
"name": "零值",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "零值比例 (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "内存占用",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
]
)
mini_histo = Image(
mini_histogram(*summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
quantile_statistics = Table(
[
{"name": "最小值", "value": summary["min"], "fmt": "fmt_numeric"},
{"name": "5百分位", "value": summary["5%"], "fmt": "fmt_numeric"},
{"name": "25百分位", "value": summary["25%"], "fmt": "fmt_numeric"},
{"name": "中位", "value": summary["50%"], "fmt": "fmt_numeric"},
{"name": "75百分位", "value": summary["75%"], "fmt": "fmt_numeric"},
{"name": "95-百分位", "value": summary["95%"], "fmt": "fmt_numeric"},
{"name": "最大值", "value": summary["max"], "fmt": "fmt_numeric"},
{"name": "极差", "value": summary["range"], "fmt": "fmt_numeric"},
{
"name": "四分位距 (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="定性统计",
)
if summary["monotonic_increase_strict"]:
monotocity = "严格递增"
elif summary["monotonic_decrease_strict"]:
monotocity = "严格递减"
elif summary["monotonic_increase"]:
monotocity = "递增"
elif summary["monotonic_decrease"]:
monotocity = "递减"
else:
monotocity = "非单调"
descriptive_statistics = Table(
[
{
"name": "标准差",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "变异系数 (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{"name": "峰度", "value": summary["kurtosis"], "fmt": "fmt_numeric"},
{"name": "均数", "value": summary["mean"], "fmt": "fmt_numeric"},
{
"name": "中位绝对偏差 (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "偏度",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class": "alert" if "skewness" in summary["warn_fields"] else "",
},
{"name": "总和", "value": summary["sum"], "fmt": "fmt_numeric"},
{"name": "方差", "value": summary["variance"], "fmt": "fmt_numeric"},
{"name": "单调性", "value": monotocity, "fmt": "fmt"},
],
name="描述性统计",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="统计",
sequence_type="grid",
)
hist = Image(
histogram(*summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=f"<strong>固定大小的直方图</strong> (bins={len(summary['histogram'][1]) - 1})",
name="直方图",
anchor_id=f"{varid}histogram",
)
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="常见值",
anchor_id=f"{varid}common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="最小10个",
anchor_id=f"{varid}firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="最大10个",
anchor_id=f"{varid}lastn",
redact=False,
),
],
sequence_type="tabs",
name="极值",
anchor_id=f"{varid}extreme_values",
)
template_variables["bottom"] = Container(
[statistics, hist, fq, evs], sequence_type="tabs", anchor_id=f"{varid}bottom",
)
return template_variables
def render_date(config: Settings, summary: Dict[str, Any]) -> Dict[str, Any]:
varid = summary["varid"]
template_variables = {}
image_format = config.plot.image_format
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Date",
summary["warnings"],
summary["description"],
)
table1 = Table(
[
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": False,
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": False,
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": False,
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": False,
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
]
)
table2 = Table(
[
{"name": "Minimum", "value": fmt(summary["min"]), "alert": False},
{"name": "Maximum", "value": fmt(summary["max"]), "alert": False},
]
)
mini_histo = Image(
mini_histogram(
config, summary["histogram"][0], summary["histogram"][1], date=True
),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container(
[info, table1, table2, mini_histo], sequence_type="grid"
)
# Bottom
bottom = Container(
[
Image(
histogram(
config, summary["histogram"][0], summary["histogram"][1], date=True
),
image_format=image_format,
alt="Histogram",
caption=f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables
def render_real(summary):
varid = summary["varid"]
template_variables = render_common(summary)
image_format = config["plot"]["image_format"].get(str)
if summary["min"] >= 0:
name = "Real number (ℝ<sub>≥0</sub>)"
else:
name = "Real number (ℝ)"
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
name,
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt",
"alert": "n_unique" in summary["warn_fields"],
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent",
"alert": "p_unique" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Infinite",
"value": summary["n_infinite"],
"fmt": "fmt",
"alert": "n_infinite" in summary["warn_fields"],
},
{
"name": "Infinite (%)",
"value": summary["p_infinite"],
"fmt": "fmt_percent",
"alert": "p_infinite" in summary["warn_fields"],
},
])
table2 = Table([
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt",
"alert": False
},
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt",
"alert": False
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt",
"alert": False
},
{
"name": "Zeros",
"value": summary["n_zeros"],
"fmt": "fmt",
"alert": "n_zeros" in summary["warn_fields"],
},
{
"name": "Zeros (%)",
"value": summary["p_zeros"],
"fmt": "fmt_percent",
"alert": "p_zeros" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
histogram_bins = 10
# TODO: replace with SmallImage...
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary, histogram_bins),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
quantile_statistics = Table(
[
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt_numeric"
},
{
"name": "5-th percentile",
"value": summary["5%"],
"fmt": "fmt_numeric"
},
{
"name": "Q1",
"value": summary["25%"],
"fmt": "fmt_numeric"
},
{
"name": "median",
"value": summary["50%"],
"fmt": "fmt_numeric"
},
{
"name": "Q3",
"value": summary["75%"],
"fmt": "fmt_numeric"
},
{
"name": "95-th percentile",
"value": summary["95%"],
"fmt": "fmt_numeric"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt_numeric"
},
{
"name": "Range",
"value": summary["range"],
"fmt": "fmt_numeric"
},
{
"name": "Interquartile range (IQR)",
"value": summary["iqr"],
"fmt": "fmt_numeric",
},
],
name="Quantile statistics",
)
descriptive_statistics = Table(
[
{
"name": "Standard deviation",
"value": summary["std"],
"fmt": "fmt_numeric",
},
{
"name": "Coefficient of variation (CV)",
"value": summary["cv"],
"fmt": "fmt_numeric",
},
{
"name": "Kurtosis",
"value": summary["kurtosis"],
"fmt": "fmt_numeric"
},
{
"name": "Mean",
"value": summary["mean"],
"fmt": "fmt_numeric"
},
{
"name": "Median Absolute Deviation (MAD)",
"value": summary["mad"],
"fmt": "fmt_numeric",
},
{
"name": "Skewness",
"value": summary["skewness"],
"fmt": "fmt_numeric",
"class":
"alert" if "skewness" in summary["warn_fields"] else "",
},
{
"name": "Sum",
"value": summary["sum"],
"fmt": "fmt_numeric"
},
{
"name": "Variance",
"value": summary["variance"],
"fmt": "fmt_numeric"
},
],
name="Descriptive statistics",
)
statistics = Container(
[quantile_statistics, descriptive_statistics],
anchor_id=f"{varid}statistics",
name="Statistics",
sequence_type="grid",
)
seqs = [
Image(
histogram(summary["histogram_data"], summary, histogram_bins),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={histogram_bins})",
name="Histogram",
anchor_id=f"{varid}histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id=f"{varid}common_values",
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id=f"{varid}firstn",
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id=f"{varid}lastn",
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id=f"{varid}extreme_values",
)
if "histogram_bins_bayesian_blocks" in summary:
histo_dyn = Image(
histogram(
summary["histogram_data"],
summary,
summary["histogram_bins_bayesian_blocks"],
),
image_format=image_format,
alt="Histogram",
caption=
'<strong>Histogram with variable size bins</strong> (bins={}, <a href="https://ui.adsabs.harvard.edu/abs/2013ApJ...764..167S/abstract" target="_blank">"bayesian blocks"</a> binning strategy used)'
.format(
fmt_array(summary["histogram_bins_bayesian_blocks"],
threshold=5)),
name="Dynamic Histogram",
anchor_id=f"{varid}dynamic_histogram",
)
seqs.append(histo_dyn)
template_variables["bottom"] = Container(
[
statistics,
Container(
seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id=f"{varid}histograms",
),
fq,
evs,
],
sequence_type="tabs",
anchor_id=f"{varid}bottom",
)
return template_variables
def render_count(config: Settings, summary: dict) -> dict:
template_variables = render_common(config, summary)
image_format = config.plot.image_format
# Top
info = VariableInfo(
summary["varid"],
summary["varname"],
"Real number (ℝ / ℝ<sub>≥0</sub>)",
summary["warnings"],
summary["description"],
)
table1 = Table([
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": False,
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": False,
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": False,
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": False,
},
])
table2 = Table([
{
"name":
"Mean",
"value":
fmt_numeric(summary["mean"], precision=config.report.precision),
"alert":
False,
},
{
"name": "Minimum",
"value": fmt_numeric(summary["min"],
precision=config.report.precision),
"alert": False,
},
{
"name": "Maximum",
"value": fmt_numeric(summary["max"],
precision=config.report.precision),
"alert": False,
},
{
"name": "Zeros",
"value": fmt(summary["n_zeros"]),
"alert": False,
},
{
"name": "Zeros (%)",
"value": fmt_percent(summary["p_zeros"]),
"alert": False,
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
])
mini_histo = Image(
mini_histogram(config, *summary["histogram"]),
image_format=image_format,
alt="Mini histogram",
)
template_variables["top"] = Container([info, table1, table2, mini_histo],
sequence_type="grid")
seqs = [
Image(
histogram(config, *summary["histogram"]),
image_format=image_format,
alt="Histogram",
caption=
f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})",
name="Histogram",
anchor_id="histogram",
)
]
fq = FrequencyTable(
template_variables["freq_table_rows"],
name="Common values",
anchor_id="common_values",
redact=False,
)
evs = Container(
[
FrequencyTable(
template_variables["firstn_expanded"],
name="Minimum 5 values",
anchor_id="firstn",
redact=False,
),
FrequencyTable(
template_variables["lastn_expanded"],
name="Maximum 5 values",
anchor_id="lastn",
redact=False,
),
],
sequence_type="tabs",
name="Extreme values",
anchor_id="extreme_values",
)
template_variables["bottom"] = Container(
[
Container(seqs,
sequence_type="tabs",
name="Histogram(s)",
anchor_id="histograms"),
fq,
evs,
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
return template_variables
def render_path(summary):
varid = summary["varid"]
n_freq_table_max = config["n_freq_table_max"].get(int)
image_format = config["plot"]["image_format"].get(str)
template_variables = render_categorical(summary)
keys = ["name", "parent", "suffix", "stem"]
for path_part in keys:
template_variables[f"freqtable_{path_part}"] = freq_table(
freqtable=summary[f"{path_part}_counts"],
n=summary["n"],
max_number_to_print=n_freq_table_max,
)
# Top
template_variables["top"].content["items"][0].content["var_type"] = "Path"
# TODO: colspan=2
# template_variables['top'].content['items'][1].content['rows'].append({'name': 'Common prefix', 'value': summary['common_prefix'], 'fmt': 'fmt'})
# { # <td>#}
# { # <div style="white-space: nowrap;overflow: hidden;text-overflow: ellipsis;max-width: 600px;">#}
# { # {{ values['common_prefix'] }}#}
# { # </div>#}
# { # </td>#}
#
# Bottom
full = FrequencyTable(
template_variables["freq_table_rows"],
name="Full",
anchor_id=f"{varid}full_frequency",
)
stem = FrequencyTable(
template_variables["freqtable_stem"],
name="Stem",
anchor_id=f"{varid}stem_frequency",
)
name = FrequencyTable(
template_variables["freqtable_name"],
name="Name",
anchor_id=f"{varid}name_frequency",
)
suffix = FrequencyTable(
template_variables["freqtable_suffix"],
name="Suffix",
anchor_id=f"{varid}suffix_frequency",
)
parent = FrequencyTable(
template_variables["freqtable_parent"],
name="Parent",
anchor_id=f"{varid}parent_frequency",
)
template_variables["bottom"].content["items"].append(full)
template_variables["bottom"].content["items"].append(stem)
template_variables["bottom"].content["items"].append(name)
template_variables["bottom"].content["items"].append(suffix)
template_variables["bottom"].content["items"].append(parent)
if "file_sizes" in summary:
file_size_histogram = Image(
histogram(summary["file_sizes"], summary,
summary["histogram_bins"]),
image_format=image_format,
alt="File size",
caption=
f"<strong>Histogram with fixed size bins of file sizes (in bytes)</strong> (bins={summary['histogram_bins']})",
name="File size",
anchor_id=f"{varid}file_size_histogram",
)
# TODO: in SequeencyItem
template_variables["bottom"].content["items"].append(
file_size_histogram)
return template_variables
def render_boolean(config: Settings, summary: dict) -> dict:
varid = summary["varid"]
n_obs_bool = config.vars.bool.n_obs
image_format = config.plot.image_format
# Prepare variables
template_variables = render_common(config, summary)
# Element composition
info = VariableInfo(
anchor_id=summary["varid"],
alerts=summary["alerts"],
var_type="Boolean",
var_name=summary["varname"],
description=summary["description"],
)
table = Table(
[
{
"name": "Distinct",
"value": fmt(summary["n_distinct"]),
"alert": "n_distinct" in summary["alert_fields"],
},
{
"name": "Distinct (%)",
"value": fmt_percent(summary["p_distinct"]),
"alert": "p_distinct" in summary["alert_fields"],
},
{
"name": "Missing",
"value": fmt(summary["n_missing"]),
"alert": "n_missing" in summary["alert_fields"],
},
{
"name": "Missing (%)",
"value": fmt_percent(summary["p_missing"]),
"alert": "p_missing" in summary["alert_fields"],
},
{
"name": "Memory size",
"value": fmt_bytesize(summary["memory_size"]),
"alert": False,
},
]
)
fqm = FrequencyTableSmall(
freq_table(
freqtable=summary["value_counts_without_nan"],
n=summary["n"],
max_number_to_print=n_obs_bool,
),
redact=False,
)
template_variables["top"] = Container([info, table, fqm], sequence_type="grid")
items: List[Renderable] = [
FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}frequency_table",
redact=False,
)
]
max_unique = config.plot.pie.max_unique
if max_unique > 0:
items.append(
Image(
pie_plot(
config,
summary["value_counts_without_nan"],
legend_kws={"loc": "upper right"},
),
image_format=image_format,
alt="Chart",
name="Chart",
anchor_id=f"{varid}pie_chart",
)
)
template_variables["bottom"] = Container(
items, sequence_type="tabs", anchor_id=f"{varid}bottom"
)
return template_variables
def render_boolean(summary):
varid = summary["varid"]
n_obs_bool = config["vars"]["bool"]["n_obs"].get(int)
image_format = config["plot"]["image_format"].get(str)
# Prepare variables
template_variables = render_common(summary)
# Element composition
info = VariableInfo(
anchor_id=summary["varid"],
warnings=summary["warnings"],
var_type="Boolean",
var_name=summary["varname"],
description=summary["description"],
)
table = Table([
{
"name": "Distinct",
"value": summary["n_distinct"],
"fmt": "fmt",
"alert": "n_distinct" in summary["warn_fields"],
},
{
"name": "Distinct (%)",
"value": summary["p_distinct"],
"fmt": "fmt_percent",
"alert": "p_distinct" in summary["warn_fields"],
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt",
"alert": "n_missing" in summary["warn_fields"],
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
"alert": "p_missing" in summary["warn_fields"],
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
"alert": False,
},
])
fqm = FrequencyTableSmall(
freq_table(
freqtable=summary["value_counts"],
n=summary["n"],
max_number_to_print=n_obs_bool,
),
redact=False,
)
template_variables["top"] = Container([info, table, fqm],
sequence_type="grid")
items = [
FrequencyTable(
template_variables["freq_table_rows"],
name="Common Values",
anchor_id=f"{varid}frequency_table",
redact=False,
)
]
max_unique = config["plot"]["pie"]["max_unique"].get(int)
if max_unique > 0:
items.append(
Image(
pie_plot(summary["value_counts"],
legend_kws={"loc": "upper right"}),
image_format=image_format,
alt="Chart",
name="Chart",
anchor_id=f"{varid}pie_chart",
))
template_variables["bottom"] = Container(items,
sequence_type="tabs",
anchor_id=f"{varid}bottom")
return template_variables
def render_date(summary):
# TODO: render common?
template_variables = {}
# Top
info = Overview(summary["varid"], summary["varname"], "Date", [])
table1 = Table([
{
"name": "Distinct count",
"value": summary["n_unique"],
"fmt": "fmt"
},
{
"name": "Unique (%)",
"value": summary["p_unique"],
"fmt": "fmt_percent"
},
{
"name": "Missing",
"value": summary["n_missing"],
"fmt": "fmt"
},
{
"name": "Missing (%)",
"value": summary["p_missing"],
"fmt": "fmt_percent",
},
{
"name": "Memory size",
"value": summary["memory_size"],
"fmt": "fmt_bytesize",
},
])
table2 = Table([
{
"name": "Minimum",
"value": summary["min"],
"fmt": "fmt"
},
{
"name": "Maximum",
"value": summary["max"],
"fmt": "fmt"
},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
# {'name': '', 'value': '', 'fmt': 'fmt'},
])
mini_histo = Image(
mini_histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
"Mini histogram",
)
template_variables["top"] = Sequence([info, table1, table2, mini_histo],
sequence_type="grid")
# Bottom
bottom = Sequence(
[
Image(
histogram(summary["histogram_data"], summary,
summary["histogram_bins"]),
alt="Histogram",
caption="Histogram",
name="Histogram",
anchor_id="{varid}histogram".format(varid=summary["varid"]),
)
],
sequence_type="tabs",
anchor_id=summary["varid"],
)
template_variables["bottom"] = bottom
return template_variables