def pandas_describe_date_1d(
config: Settings, series: pd.Series, summary: dict
) -> Tuple[Settings, pd.Series, dict]:
"""Describe a date series.
Args:
config: report Settings object
series: The Series to describe.
summary: The dict containing the series description so far.
Returns:
A dict containing calculated series description values.
"""
summary.update(
{
"min": pd.Timestamp.to_pydatetime(series.min()),
"max": pd.Timestamp.to_pydatetime(series.max()),
}
)
summary["range"] = summary["max"] - summary["min"]
values = series.values.astype(np.int64) // 10**9
if config.vars.num.chi_squared_threshold > 0.0:
summary["chi_squared"] = chi_square(values)
summary.update(histogram_compute(config, values, summary["n_distinct"]))
return config, values, summary
def pandas_describe_categorical_1d(
config: Settings, series: pd.Series,
summary: dict) -> Tuple[Settings, pd.Series, dict]:
"""Describe a categorical series.
Args:
config: report Settings object
series: The Series to describe.
summary: The dict containing the series description so far.
Returns:
A dict containing calculated series description values.
"""
# Make sure we deal with strings (Issue #100)
series = series.astype(str)
# Only run if at least 1 non-missing value
value_counts = summary["value_counts_without_nan"]
value_counts.index = value_counts.index.astype(str)
redact = config.vars.cat.redact
if not redact:
summary.update({"first_rows": series.head(5)})
chi_squared_threshold = config.vars.num.chi_squared_threshold
if chi_squared_threshold > 0.0:
summary["chi_squared"] = chi_square(histogram=value_counts.values)
if config.vars.cat.length:
summary.update(length_summary_vc(value_counts))
summary.update(
histogram_compute(
config,
summary["length_histogram"].index.values,
len(summary["length_histogram"]),
name="histogram_length",
weights=summary["length_histogram"].values,
))
if config.vars.cat.characters:
summary.update(unicode_summary_vc(value_counts))
if config.vars.cat.words:
summary.update(
word_summary_vc(value_counts, config.vars.cat.stop_words))
return config, series, summary
def pandas_describe_file_1d(
config: Settings, series: pd.Series, summary: dict
) -> Tuple[Settings, pd.Series, dict]:
if series.hasnans:
raise ValueError("May not contain NaNs")
if not hasattr(series, "str"):
raise ValueError("series should have .str accessor")
summary.update(file_summary(series))
summary.update(
histogram_compute(
config,
summary["file_size"],
summary["file_size"].nunique(),
name="histogram_file_size",
)
)
return config, series, summary
def pandas_describe_numeric_1d(
config: Settings, series: pd.Series, summary: dict
) -> Tuple[Settings, pd.Series, dict]:
"""Describe a numeric series.
Args:
config: report Settings object
series: The Series to describe.
summary: The dict containing the series description so far.
Returns:
A dict containing calculated series description values.
"""
chi_squared_threshold = config.vars.num.chi_squared_threshold
quantiles = config.vars.num.quantiles
value_counts = summary["value_counts_without_nan"]
negative_index = value_counts.index < 0
summary["n_negative"] = value_counts.loc[negative_index].sum()
summary["p_negative"] = summary["n_negative"] / summary["n"]
infinity_values = [np.inf, -np.inf]
infinity_index = value_counts.index.isin(infinity_values)
summary["n_infinite"] = value_counts.loc[infinity_index].sum()
summary["n_zeros"] = 0
if 0 in value_counts.index:
summary["n_zeros"] = value_counts.loc[0]
stats = summary
if isinstance(series.dtype, _IntegerDtype):
stats.update(numeric_stats_pandas(series))
present_values = series.astype(str(series.dtype).lower())
finite_values = present_values
else:
present_values = series.values
finite_values = present_values[np.isfinite(present_values)]
stats.update(numeric_stats_numpy(present_values, series, summary))
stats.update(
{
"mad": mad(present_values),
}
)
if chi_squared_threshold > 0.0:
stats["chi_squared"] = chi_square(finite_values)
stats["range"] = stats["max"] - stats["min"]
stats.update(
{
f"{percentile:.0%}": value
for percentile, value in series.quantile(quantiles).to_dict().items()
}
)
stats["iqr"] = stats["75%"] - stats["25%"]
stats["cv"] = stats["std"] / stats["mean"] if stats["mean"] else np.NaN
stats["p_zeros"] = stats["n_zeros"] / summary["n"]
stats["p_infinite"] = summary["n_infinite"] / summary["n"]
stats["monotonic_increase"] = series.is_monotonic_increasing
stats["monotonic_decrease"] = series.is_monotonic_decreasing
stats["monotonic_increase_strict"] = (
stats["monotonic_increase"] and series.is_unique
)
stats["monotonic_decrease_strict"] = (
stats["monotonic_decrease"] and series.is_unique
)
if summary["monotonic_increase_strict"]:
stats["monotonic"] = 2
elif summary["monotonic_decrease_strict"]:
stats["monotonic"] = -2
elif summary["monotonic_increase"]:
stats["monotonic"] = 1
elif summary["monotonic_decrease"]:
stats["monotonic"] = -1
else:
stats["monotonic"] = 0
stats.update(
histogram_compute(
config,
value_counts[~infinity_index].index.values,
summary["n_distinct"],
weights=value_counts[~infinity_index].values,
)
)
return config, series, stats