def test_effectsize_power(): # example and results from PASS documentation n_groups = 3 means = [527.86, 660.43, 649.14] vars_ = 107.4304**2 nobs = 12 es = effectsize_oneway(means, vars_, nobs, use_var="equal", ddof_between=0) es = np.sqrt(es) alpha = 0.05 power = 0.8 nobs_t = nobs * n_groups kwds = { 'effect_size': es, 'nobs': nobs_t, 'alpha': alpha, 'power': power, 'k_groups': n_groups } from statsmodels.stats.power import FTestAnovaPower res_pow = 0.8251 res_es = 0.559 kwds_ = kwds.copy() del kwds_['power'] p = FTestAnovaPower().power(**kwds_) assert_allclose(p, res_pow, atol=0.0001) assert_allclose(es, res_es, atol=0.0006) # example unequal sample sizes nobs = np.array([15, 9, 9]) kwds['nobs'] = nobs es = effectsize_oneway(means, vars_, nobs, use_var="equal", ddof_between=0) es = np.sqrt(es) kwds['effect_size'] = es p = FTestAnovaPower().power(**kwds_) res_pow = 0.8297 res_es = 0.590 assert_allclose(p, res_pow, atol=0.005) # lower than print precision assert_allclose(es, res_es, atol=0.0006)
def test_simulate_equivalence(): # regression test, needs large k_mc to be reliable k_groups = 4 k_repl = 10 nobs = np.array([10, 12, 13, 15]) * k_repl means = np.array([-1, 0, 0, 1]) * 0.12 vars_ = np.array([1, 2, 3, 4]) nobs_t = nobs.sum() eps = 0.0191 * 10 opt_var = ["unequal", "equal", "bf"] k_mc = 100 np.random.seed(987126) res_mc = smo.simulate_power_equivalence_oneway(means, nobs, eps, vars_=vars_, k_mc=k_mc, trim_frac=0.1, options_var=opt_var, margin_type="wellek") frac_reject = (res_mc.pvalue <= 0.05).sum(0) / k_mc assert_allclose(frac_reject, [0.17, 0.18, 0.14], atol=0.001) # result with k_mc = 10000 is [0.1466, 0.1871, 0.1606] # similar to asy below, but not very close for all es_alt_li = [] for uv in opt_var: es = effectsize_oneway(means, vars_, nobs, use_var=uv) es_alt_li.append(es) # compute asy power as comparison margin = wellek_to_f2(eps, k_groups) pow_ = [ power_equivalence_oneway(es_, margin, nobs_t, n_groups=k_groups, df=None, alpha=0.05, margin_type="f2") for es_ in es_alt_li ] # regression test numbers assert_allclose(pow_, [0.147749, 0.173358, 0.177412], atol=0.007)
def coehen_f(data, groups, metric): """"Cohens f, for calculating the effect size for anova power analysis. Args: data (DataFrame): DataFrame with columns 'groups' and 'metric'. groups (str): Name of the groups column in the data, i.e. the groups for the anova test. metric (str): Name of the metric column in the data, i.e. the dependant variable. Returns: cohens_f (float): Cohens f effect size. """ # Get the means and variance of the data means = data.groupby( [groups], sort=False).apply(lambda x: mean(x[metric])).to_list() variances = data.groupby( [groups], sort=False).apply(lambda x: variance(x[metric])).to_list() # Calculate effect size cohens_f = effectsize_oneway(means, variances, len(data), use_var='equal') return cohens_f
def test_ols_noncentrality(self): k = self.k_groups res_ols = OLS(self.y, self.ex).fit() nobs_t = res_ols.model.nobs # constraint c_equal = -np.eye(k)[1:] c_equal[:, 0] = 1 v = np.zeros(c_equal.shape[0]) # noncentrality at estimated parameters wt = res_ols.wald_test(c_equal, scalar=True) df_num, df_denom = wt.df_num, wt.df_denom cov_p = res_ols.cov_params() nc_wt = wald_test_noncent_generic(res_ols.params, c_equal, v, cov_p, diff=None, joint=True) assert_allclose(nc_wt, wt.statistic * wt.df_num, rtol=1e-13) nc_wt2 = wald_test_noncent(res_ols.params, c_equal, v, res_ols, diff=None, joint=True) assert_allclose(nc_wt2, nc_wt, rtol=1e-13) es_ols = nc_wt / nobs_t es_oneway = smo.effectsize_oneway(res_ols.params, res_ols.scale, self.nobs, use_var="equal") assert_allclose(es_ols, es_oneway, rtol=1e-13) alpha = 0.05 pow_ols = smpwr.ftest_power(np.sqrt(es_ols), df_denom, df_num, alpha, ncc=1) pow_oneway = smpwr.ftest_anova_power(np.sqrt(es_oneway), nobs_t, alpha, k_groups=k, df=None) assert_allclose(pow_ols, pow_oneway, rtol=1e-13) # noncentrality at other params params_alt = res_ols.params * 0.75 # compute constraint value so we can get noncentrality from wald_test v_off = _offset_constraint(c_equal, res_ols.params, params_alt) wt_off = res_ols.wald_test((c_equal, v + v_off), scalar=True) nc_wt_off = wald_test_noncent_generic(params_alt, c_equal, v, cov_p, diff=None, joint=True) assert_allclose(nc_wt_off, wt_off.statistic * wt_off.df_num, rtol=1e-13) # check vectorized version, joint=False nc_wt_vec = wald_test_noncent_generic(params_alt, c_equal, v, cov_p, diff=None, joint=False) for i in range(c_equal.shape[0]): nc_wt_i = wald_test_noncent_generic( params_alt, c_equal[i:i + 1], # noqa v[i:i + 1], cov_p, diff=None, # noqa joint=False) assert_allclose(nc_wt_vec[i], nc_wt_i, rtol=1e-13)
def test_equivalence_welch(self): # reference numbers from Jan and Shieh 2019, p. 6 means = self.means nobs = self.nobs stds = self.stds n_groups = self.n_groups vars_ = stds**2 eps = 0.5 res0 = anova_generic(means, vars_, nobs, use_var="unequal", welch_correction=False) f_stat = res0.statistic res = equivalence_oneway_generic(f_stat, n_groups, nobs.sum(), eps, res0.df, alpha=0.05, margin_type="wellek") assert_allclose(res.pvalue, 0.0110, atol=0.001) assert_allclose(res.df, [3.0, 22.6536], atol=0.0006) # agreement for Welch f-stat looks too low b/c welch_correction=False assert_allclose(f_stat, 0.1102, atol=0.007) res = equivalence_oneway(self.data, eps, use_var="unequal", margin_type="wellek") assert_allclose(res.pvalue, 0.0110, atol=1e-4) assert_allclose(res.df, [3.0, 22.6536], atol=0.0006) assert_allclose(res.f_stat, 0.1102, atol=1e-4) # 0.007) # check post-hoc power, JS p. 6 pow_ = _power_equivalence_oneway_emp(f_stat, n_groups, nobs, eps, res0.df) assert_allclose(pow_, 0.1552, atol=0.007) pow_ = power_equivalence_oneway(eps, eps, nobs.sum(), n_groups=n_groups, df=None, alpha=0.05, margin_type="wellek") assert_allclose(pow_, 0.05, atol=1e-13) nobs_t = nobs.sum() es = effectsize_oneway(means, vars_, nobs, use_var="unequal") es = np.sqrt(es) es_w0 = f2_to_wellek(es**2, n_groups) es_w = np.sqrt(fstat_to_wellek(f_stat, n_groups, nobs_t / n_groups)) pow_ = power_equivalence_oneway(es_w, eps, nobs_t, n_groups=n_groups, df=None, alpha=0.05, margin_type="wellek") assert_allclose(pow_, 0.1552, atol=0.007) assert_allclose(es_w0, es_w, atol=0.007) margin = wellek_to_f2(eps, n_groups) pow_ = power_equivalence_oneway(es**2, margin, nobs_t, n_groups=n_groups, df=None, alpha=0.05, margin_type="f2") assert_allclose(pow_, 0.1552, atol=0.007)