def describe_numeric_1d(series, **kwargs):
"""Compute summary statistics of a numerical (`TYPE_NUM`) variable (a Series).
Also create histograms (mini an full) of its distribution.
Parameters
----------
series : Series
The variable to describe.
Returns
-------
Series
The description of the variable as a Series with index being stats keys.
"""
# Format a number as a percentage. For example 0.25 will be turned to 25%.
_percentile_format = "{:.0%}"
stats = dict()
stats['type'] = base.TYPE_NUM
stats['mean'] = series.mean()
stats['std'] = series.std()
stats['variance'] = series.var()
stats['min'] = series.min()
stats['max'] = series.max()
stats['range'] = stats['max'] - stats['min']
# TODO: Remove this "lazy" operation
_series_no_na = series.dropna()
for percentile in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
# The dropna() is a workaround for https://github.com/pydata/pandas/issues/13098
stats[_percentile_format.format(percentile)] = _series_no_na.quantile(
percentile)
stats['iqr'] = stats['75%'] - stats['25%']
stats['kurtosis'] = delayed(sp.stats.kurtosis)(series)
stats['skewness'] = delayed(float)(dask_stats.skew(series.to_dask_array()))
stats['sum'] = series.sum()
stats['mad'] = series.sub(series.mean()).abs().mean()
# removed conditional for testing (no purpose was seen)
# stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
stats['cv'] = stats['std'] / stats['mean']
stats['n_zeros'] = (series.size - delayed(np.count_nonzero)(series))
stats['p_zeros'] = stats['n_zeros'] * 1.0 / series.size
# Histograms
# TODO: optimize histogram and mini_histogram calls (a big overlap in computation)
stats['histogram'] = histogram(series, **kwargs)
stats['mini_histogram'] = mini_histogram(series, **kwargs)
return stats
def process(sensorId):
print("Processing %s" % sensorId)
df = dd.read_csv('data-processed/%s/*.csv' % (sensorId),
usecols=["W1", "W2", "W3"])
x = (df.W1.abs() + df.W2.abs() + df.W3.abs())
m, std, kurtosis, skew, minimum, maximum = [
x.mean(),
x.std(),
stats.kurtosis(x),
stats.skew(x),
x.min(),
x.max()
]
result = dask.compute(m, std, kurtosis, skew, minimum, maximum)
print(result)
return result
def calc_cont_col(srs: dd.Series, bins: int) -> Dict[str, Any]:
"""
Computations for a numerical column in plot(df)
Parameters
----------
srs
srs over which to compute the barchart and insights
bins
number of bins in the bar chart
"""
# dictionary of data for the histogram and related insights
data: Dict[str, Any] = {}
## if cfg.insight.missing_enable:
data["npres"] = srs.shape[0]
## if cfg.insight.infinity_enable:
data["ninf"] = srs.isin({np.inf, -np.inf}).sum()
# remove infinite values
srs = srs[~srs.isin({np.inf, -np.inf})]
## if cfg.hist_enable or config.insight.uniform_enable or cfg.insight.normal_enable:
## bins = cfg.hist_bins
data["hist"] = da.histogram(srs, bins=bins, range=[srs.min(), srs.max()])
## if cfg.insight.uniform_enable:
data["chisq"] = chisquare(data["hist"][0])
## if cfg.insight.normal_enable
data["norm"] = normaltest(data["hist"][0])
## if cfg.insight.negative_enable:
data["nneg"] = (srs < 0).sum()
## if cfg.insight.skew_enabled:
data["skew"] = skew(srs)
## if cfg.insight.unique_enabled:
data["nuniq"] = srs.nunique()
## if cfg.insight.zero_enabled:
data["nzero"] = (srs == 0).sum()
return data
def cont_comps(srs: dd.Series, cfg: Config) -> Dict[str, Any]:
"""
All computations required for plot(df, Continuous)
"""
# pylint: disable=too-many-branches
data: Dict[str, Any] = {}
if cfg.stats.enable or cfg.hist.enable:
data["nrows"] = srs.shape[0] # total rows
srs = srs.dropna()
if cfg.stats.enable:
data["npres"] = srs.shape[0] # number of present (not null) values
srs = srs[~srs.isin({np.inf, -np.inf})] # remove infinite values
if cfg.hist.enable or cfg.qqnorm.enable and cfg.insight.enable:
data["hist"] = da.histogram(srs, cfg.hist.bins, (srs.min(), srs.max()))
if cfg.insight.enable:
data["norm"] = normaltest(data["hist"][0])
if cfg.hist.enable and cfg.insight.enable:
data["chisq"] = chisquare(data["hist"][0])
# compute only the required amount of quantiles
if cfg.qqnorm.enable:
data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99))
elif cfg.stats.enable:
data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95])
elif cfg.box.enable:
data["qntls"] = srs.quantile([0.25, 0.5, 0.75])
if cfg.stats.enable or cfg.hist.enable and cfg.insight.enable:
data["skew"] = skew(srs)
if cfg.stats.enable or cfg.qqnorm.enable:
data["mean"] = srs.mean()
data["std"] = srs.std()
if cfg.stats.enable:
data["min"] = srs.min()
data["max"] = srs.max()
data["nreals"] = srs.shape[0]
data["nzero"] = (srs == 0).sum()
data["nneg"] = (srs < 0).sum()
data["kurt"] = kurtosis(srs)
data["mem_use"] = srs.memory_usage(deep=True)
# compute the density histogram
if cfg.kde.enable:
# To avoid the singular matrix problem, gaussian_kde needs a non-zero std.
if not math.isclose(
dask.compute(data["min"])[0],
dask.compute(data["max"])[0]):
data["dens"] = da.histogram(srs,
cfg.kde.bins, (srs.min(), srs.max()),
density=True)
# gaussian kernel density estimate
data["kde"] = gaussian_kde(
srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])),
meta=srs))
else:
data["kde"] = None
if cfg.box.enable:
data.update(_calc_box(srs, data["qntls"], cfg))
if cfg.value_table.enable:
value_counts = srs.value_counts(sort=False)
if cfg.stats.enable:
data["nuniq"] = value_counts.shape[0]
data["value_table"] = value_counts.nlargest(cfg.value_table.ngroups)
elif cfg.stats.enable:
data["nuniq"] = srs.nunique_approx()
return data
def cont_comps(srs: dd.Series, bins: int) -> Dict[str, Any]:
"""
This function aggregates all of the computations required for plot(df, Continuous())
Parameters
----------
srs
one numerical column
bins
the number of bins in the histogram
"""
data: Dict[str, Any] = {}
## if cfg.stats_enable or cfg.hist_enable or
# calculate the total number of rows then drop the missing values
data["nrows"] = srs.shape[0]
srs = srs.dropna()
## if cfg.stats_enable
# number of not null (present) values
data["npres"] = srs.shape[0]
# remove infinite values
srs = srs[~srs.isin({np.inf, -np.inf})]
# shared computations
## if cfg.stats_enable or cfg.hist_enable or cfg.qqplot_enable and cfg.insights_enable:
data["min"], data["max"] = srs.min(), srs.max()
## if cfg.hist_enable or cfg.qqplot_enable and cfg.ingsights_enable:
data["hist"] = da.histogram(srs,
bins=bins,
range=[data["min"], data["max"]])
## if cfg.insights_enable and (cfg.qqplot_enable or cfg.hist_enable):
data["norm"] = normaltest(data["hist"][0])
## if cfg.qqplot_enable
data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99))
## elif cfg.stats_enable
## data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95])
## elif cfg.boxplot_enable
## data["qntls"] = srs.quantile([0.25, 0.5, 0.75])
## if cfg.stats_enable or cfg.hist_enable and cfg.insights_enable:
data["skew"] = skew(srs)
# if cfg.stats_enable
data["nuniq"] = srs.nunique()
data["nreals"] = srs.shape[0]
data["nzero"] = (srs == 0).sum()
data["nneg"] = (srs < 0).sum()
data["mean"] = srs.mean()
data["std"] = srs.std()
data["kurt"] = kurtosis(srs)
data["mem_use"] = srs.memory_usage(deep=True)
## if cfg.hist_enable and cfg.insight_enable
data["chisq"] = chisquare(data["hist"][0])
# compute the density histogram
data["dens"] = da.histogram(srs,
bins=bins,
range=[data["min"], data["max"]],
density=True)
# gaussian kernel density estimate
data["kde"] = gaussian_kde(
srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])),
meta=srs))
## if cfg.box_enable
data.update(calc_box(srs, data["qntls"]))
return data