def process(sensorId):
print("Processing %s" % sensorId)
df = dd.read_csv('data-processed/%s/*.csv' % (sensorId),
usecols=["W1", "W2", "W3"])
x = (df.W1.abs() + df.W2.abs() + df.W3.abs())
m, std, kurtosis, skew, minimum, maximum = [
x.mean(),
x.std(),
stats.kurtosis(x),
stats.skew(x),
x.min(),
x.max()
]
result = dask.compute(m, std, kurtosis, skew, minimum, maximum)
print(result)
return result
def cont_comps(srs: dd.Series, cfg: Config) -> Dict[str, Any]:
"""
All computations required for plot(df, Continuous)
"""
# pylint: disable=too-many-branches
data: Dict[str, Any] = {}
if cfg.stats.enable or cfg.hist.enable:
data["nrows"] = srs.shape[0] # total rows
srs = srs.dropna()
if cfg.stats.enable:
data["npres"] = srs.shape[0] # number of present (not null) values
srs = srs[~srs.isin({np.inf, -np.inf})] # remove infinite values
if cfg.hist.enable or cfg.qqnorm.enable and cfg.insight.enable:
data["hist"] = da.histogram(srs, cfg.hist.bins, (srs.min(), srs.max()))
if cfg.insight.enable:
data["norm"] = normaltest(data["hist"][0])
if cfg.hist.enable and cfg.insight.enable:
data["chisq"] = chisquare(data["hist"][0])
# compute only the required amount of quantiles
if cfg.qqnorm.enable:
data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99))
elif cfg.stats.enable:
data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95])
elif cfg.box.enable:
data["qntls"] = srs.quantile([0.25, 0.5, 0.75])
if cfg.stats.enable or cfg.hist.enable and cfg.insight.enable:
data["skew"] = skew(srs)
if cfg.stats.enable or cfg.qqnorm.enable:
data["mean"] = srs.mean()
data["std"] = srs.std()
if cfg.stats.enable:
data["min"] = srs.min()
data["max"] = srs.max()
data["nreals"] = srs.shape[0]
data["nzero"] = (srs == 0).sum()
data["nneg"] = (srs < 0).sum()
data["kurt"] = kurtosis(srs)
data["mem_use"] = srs.memory_usage(deep=True)
# compute the density histogram
if cfg.kde.enable:
# To avoid the singular matrix problem, gaussian_kde needs a non-zero std.
if not math.isclose(
dask.compute(data["min"])[0],
dask.compute(data["max"])[0]):
data["dens"] = da.histogram(srs,
cfg.kde.bins, (srs.min(), srs.max()),
density=True)
# gaussian kernel density estimate
data["kde"] = gaussian_kde(
srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])),
meta=srs))
else:
data["kde"] = None
if cfg.box.enable:
data.update(_calc_box(srs, data["qntls"], cfg))
if cfg.value_table.enable:
value_counts = srs.value_counts(sort=False)
if cfg.stats.enable:
data["nuniq"] = value_counts.shape[0]
data["value_table"] = value_counts.nlargest(cfg.value_table.ngroups)
elif cfg.stats.enable:
data["nuniq"] = srs.nunique_approx()
return data
def cont_comps(srs: dd.Series, bins: int) -> Dict[str, Any]:
"""
This function aggregates all of the computations required for plot(df, Continuous())
Parameters
----------
srs
one numerical column
bins
the number of bins in the histogram
"""
data: Dict[str, Any] = {}
## if cfg.stats_enable or cfg.hist_enable or
# calculate the total number of rows then drop the missing values
data["nrows"] = srs.shape[0]
srs = srs.dropna()
## if cfg.stats_enable
# number of not null (present) values
data["npres"] = srs.shape[0]
# remove infinite values
srs = srs[~srs.isin({np.inf, -np.inf})]
# shared computations
## if cfg.stats_enable or cfg.hist_enable or cfg.qqplot_enable and cfg.insights_enable:
data["min"], data["max"] = srs.min(), srs.max()
## if cfg.hist_enable or cfg.qqplot_enable and cfg.ingsights_enable:
data["hist"] = da.histogram(srs,
bins=bins,
range=[data["min"], data["max"]])
## if cfg.insights_enable and (cfg.qqplot_enable or cfg.hist_enable):
data["norm"] = normaltest(data["hist"][0])
## if cfg.qqplot_enable
data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99))
## elif cfg.stats_enable
## data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95])
## elif cfg.boxplot_enable
## data["qntls"] = srs.quantile([0.25, 0.5, 0.75])
## if cfg.stats_enable or cfg.hist_enable and cfg.insights_enable:
data["skew"] = skew(srs)
# if cfg.stats_enable
data["nuniq"] = srs.nunique()
data["nreals"] = srs.shape[0]
data["nzero"] = (srs == 0).sum()
data["nneg"] = (srs < 0).sum()
data["mean"] = srs.mean()
data["std"] = srs.std()
data["kurt"] = kurtosis(srs)
data["mem_use"] = srs.memory_usage(deep=True)
## if cfg.hist_enable and cfg.insight_enable
data["chisq"] = chisquare(data["hist"][0])
# compute the density histogram
data["dens"] = da.histogram(srs,
bins=bins,
range=[data["min"], data["max"]],
density=True)
# gaussian kernel density estimate
data["kde"] = gaussian_kde(
srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])),
meta=srs))
## if cfg.box_enable
data.update(calc_box(srs, data["qntls"]))
return data