def process(sensorId): print("Processing %s" % sensorId) df = dd.read_csv('data-processed/%s/*.csv' % (sensorId), usecols=["W1", "W2", "W3"]) x = (df.W1.abs() + df.W2.abs() + df.W3.abs()) m, std, kurtosis, skew, minimum, maximum = [ x.mean(), x.std(), stats.kurtosis(x), stats.skew(x), x.min(), x.max() ] result = dask.compute(m, std, kurtosis, skew, minimum, maximum) print(result) return result
def cont_comps(srs: dd.Series, cfg: Config) -> Dict[str, Any]: """ All computations required for plot(df, Continuous) """ # pylint: disable=too-many-branches data: Dict[str, Any] = {} if cfg.stats.enable or cfg.hist.enable: data["nrows"] = srs.shape[0] # total rows srs = srs.dropna() if cfg.stats.enable: data["npres"] = srs.shape[0] # number of present (not null) values srs = srs[~srs.isin({np.inf, -np.inf})] # remove infinite values if cfg.hist.enable or cfg.qqnorm.enable and cfg.insight.enable: data["hist"] = da.histogram(srs, cfg.hist.bins, (srs.min(), srs.max())) if cfg.insight.enable: data["norm"] = normaltest(data["hist"][0]) if cfg.hist.enable and cfg.insight.enable: data["chisq"] = chisquare(data["hist"][0]) # compute only the required amount of quantiles if cfg.qqnorm.enable: data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99)) elif cfg.stats.enable: data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95]) elif cfg.box.enable: data["qntls"] = srs.quantile([0.25, 0.5, 0.75]) if cfg.stats.enable or cfg.hist.enable and cfg.insight.enable: data["skew"] = skew(srs) if cfg.stats.enable or cfg.qqnorm.enable: data["mean"] = srs.mean() data["std"] = srs.std() if cfg.stats.enable: data["min"] = srs.min() data["max"] = srs.max() data["nreals"] = srs.shape[0] data["nzero"] = (srs == 0).sum() data["nneg"] = (srs < 0).sum() data["kurt"] = kurtosis(srs) data["mem_use"] = srs.memory_usage(deep=True) # compute the density histogram if cfg.kde.enable: # To avoid the singular matrix problem, gaussian_kde needs a non-zero std. if not math.isclose( dask.compute(data["min"])[0], dask.compute(data["max"])[0]): data["dens"] = da.histogram(srs, cfg.kde.bins, (srs.min(), srs.max()), density=True) # gaussian kernel density estimate data["kde"] = gaussian_kde( srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])), meta=srs)) else: data["kde"] = None if cfg.box.enable: data.update(_calc_box(srs, data["qntls"], cfg)) if cfg.value_table.enable: value_counts = srs.value_counts(sort=False) if cfg.stats.enable: data["nuniq"] = value_counts.shape[0] data["value_table"] = value_counts.nlargest(cfg.value_table.ngroups) elif cfg.stats.enable: data["nuniq"] = srs.nunique_approx() return data
def cont_comps(srs: dd.Series, bins: int) -> Dict[str, Any]: """ This function aggregates all of the computations required for plot(df, Continuous()) Parameters ---------- srs one numerical column bins the number of bins in the histogram """ data: Dict[str, Any] = {} ## if cfg.stats_enable or cfg.hist_enable or # calculate the total number of rows then drop the missing values data["nrows"] = srs.shape[0] srs = srs.dropna() ## if cfg.stats_enable # number of not null (present) values data["npres"] = srs.shape[0] # remove infinite values srs = srs[~srs.isin({np.inf, -np.inf})] # shared computations ## if cfg.stats_enable or cfg.hist_enable or cfg.qqplot_enable and cfg.insights_enable: data["min"], data["max"] = srs.min(), srs.max() ## if cfg.hist_enable or cfg.qqplot_enable and cfg.ingsights_enable: data["hist"] = da.histogram(srs, bins=bins, range=[data["min"], data["max"]]) ## if cfg.insights_enable and (cfg.qqplot_enable or cfg.hist_enable): data["norm"] = normaltest(data["hist"][0]) ## if cfg.qqplot_enable data["qntls"] = srs.quantile(np.linspace(0.01, 0.99, 99)) ## elif cfg.stats_enable ## data["qntls"] = srs.quantile([0.05, 0.25, 0.5, 0.75, 0.95]) ## elif cfg.boxplot_enable ## data["qntls"] = srs.quantile([0.25, 0.5, 0.75]) ## if cfg.stats_enable or cfg.hist_enable and cfg.insights_enable: data["skew"] = skew(srs) # if cfg.stats_enable data["nuniq"] = srs.nunique() data["nreals"] = srs.shape[0] data["nzero"] = (srs == 0).sum() data["nneg"] = (srs < 0).sum() data["mean"] = srs.mean() data["std"] = srs.std() data["kurt"] = kurtosis(srs) data["mem_use"] = srs.memory_usage(deep=True) ## if cfg.hist_enable and cfg.insight_enable data["chisq"] = chisquare(data["hist"][0]) # compute the density histogram data["dens"] = da.histogram(srs, bins=bins, range=[data["min"], data["max"]], density=True) # gaussian kernel density estimate data["kde"] = gaussian_kde( srs.map_partitions(lambda x: x.sample(min(1000, x.shape[0])), meta=srs)) ## if cfg.box_enable data.update(calc_box(srs, data["qntls"])) return data