def test_tukeyRangeTest_pResult(self): x1, x2, x3 = [1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15] results = tukey_range_test(x1, x2, x3) model = pairwise_tukeyhsd(x1 + x2 + x3, groups=[0] * 5 + [1] * 5 + [2] * 5) p_vals = psturng(np.abs(model.meandiffs / model.std_pairs), len(model.groupsunique), model.df_total) for i in range(3): assert pytest.approx(p_vals[i]) == results[i][2]
def _calc_firing_rate(self, num_peaks: pd.DataFrame, epoch: str = "All_cells"): """ Sum all indices of peaks to find the average firing rate of cells in the three epochs :return: """ # Remove silent cells from comparison split_data = num_peaks.stack() mc = MultiComparison(split_data.values, split_data.index.get_level_values(1).values) try: res = mc.tukeyhsd() except ValueError: aprint("<yellow>Failed during the p-value calculation.</yellow>") else: print(res) print( f"P-values ({epoch}, number of cells: {split_data.shape[0] // 3}):", psturng( np.abs(res.meandiffs / res.std_pairs), len(res.groupsunique), res.df_total, ), ) finally: print(split_data.mean(level=1))
def test_handful_to_known_values(self): cases = [ (0.71499578726111435, 67, 956.70742488392386, 5.0517658443070692), (0.42974234855067672, 16, 723.50261736502318, 3.3303582093701354), (0.94936429359548424, 2, 916.1867328010926, 2.7677975546417244), (0.85357381770725038, 66, 65.67055060832368, 5.5647438108270109), (0.87372108021900929, 74, 626.42369474993632, 5.5355540570701107), (0.53891960564713726, 49, 862.63799438485785, 4.5108645923377146), (0.98818659555664567, 18, 36.269686711464274, 6.0906643750886156), (0.53031994896037626, 50, 265.29558652727917, 4.5179640079726795), (0.7318857887397332, 59, 701.41497552251201, 4.9980139875409915), (0.65332019368982697, 61, 591.01183664195912, 4.8706581766706893), (0.55403221657248558, 77, 907.34156725405194, 4.8786135917984632), (0.30783916857266003, 83, 82.446923487980882, 4.4396401242858294), (0.29321720242415661, 16, 709.64382575553009, 3.0304277540702729), (0.27146478168880306, 31, 590.00594683574172, 3.5870031664477215), (0.67348796958433776, 81, 608.02706111127657, 5.1096199974432936), (0.32774393945968938, 18, 17.706224399250839, 3.2119038163765432), (0.7081637474795982, 72, 443.10678914889695, 5.0990030889410649), (0.33354939276757861, 47, 544.0772192199048, 4.0613352964193279), (0.60412143947363051, 36, 895.83526933271548, 4.381717596850172), (0.88739052300665977, 77, 426.03665511558262, 5.6333929480341309) ] for p, r, v, q in cases: assert_almost_equal(1. - p, psturng(q, r, v), 5)
def analyze_tukey_dict(tukey_dict): metric_names = list(tukey_dict.keys()) model_names = list(tukey_dict[metric_names[0]].groupsunique) tukey_analysis_dict = {} for i, metric in zip(range(len(metric_names)), metric_names): tukey_res = tukey_dict[metric] tukey_df = pd.DataFrame(data=tukey_res._results_table.data[1:], columns=tukey_res._results_table.data[0]) fwer = tukey_res._results_table.title[tukey_res._results_table.title. find('FWER'):] tukey_df['p value' + fwer] = psturng( np.abs(tukey_res.meandiffs / tukey_res.std_pairs), len(tukey_res.groupsunique), tukey_res.df_total) reject_df = pd.DataFrame(0, index=model_names, columns=model_names) comp_df = pd.DataFrame(0, index=model_names, columns=model_names) for _, row in tukey_df.iterrows(): if row['reject'] == True: if row['meandiff'] < 0: comp_df.loc[row['group1']][row['group2']] = 1 else: comp_df.loc[row['group2']][row['group1']] = 1 reject_df.loc[row['group1']][row['group2']] = row['reject'] tukey_df.columns.name = metric reject_df.columns.name = metric comp_df.columns.name = metric tukey_analysis_dict[metric] = (tukey_df, reject_df, comp_df) return tukey_analysis_dict
def get_tukey(exp, df_all, measure): # Tukey posthoc analysis # See https://jpktd.blogspot.com/2013/03/multiple-comparison-and-tukey-hsd-or_25.html # And https://code.google.com/archive/p/qsturng-py/ # And https://stackoverflow.com/questions/48200699/how-can-i-get-p-values-of-each-group-comparison-when-applying-the-tukey-s-hones # q, res_table, std_pairs, etc can be found from print(dir(result)) which will list all possible calculations if len(df_all.groupby('strain').count()) >= 3: df_tukey = df_all[np.isfinite(df_all[measure])] mc = MultiComparison(df_tukey[measure], df_tukey['strain']) result = mc.tukeyhsd() p = psturng(np.abs(result.meandiffs / result.std_pairs), len(result.groupsunique), result.df_total) df_pairs = pd.DataFrame({ 'group1': [ result._results_table[1][0], result._results_table[2][0], result._results_table[3][0] ], 'group2': [ result._results_table[1][1], result._results_table[2][1], result._results_table[3][1] ], 'p_value': [np.around(p[0], 4), np.around(p[1], 4), np.around(p[2], 4)] }) else: df_pairs = pd.DataFrame({'group1': [], 'group2': [], 'p_value': []}) file_out = exp.name + '_coupling_' + measure + '_' + '_tukey_' + '.csv' pfiles.save_csv(df_pairs, file_out, exp.dir_tukey, False) return df_pairs
def pairwise_tuckey(aov, categories): categories = np.array(aov.group_stats.index) n_groups = len(categories) gnobs = aov.group_stats["count"].to_numpy() gmeans = (aov.group_stats["sum"] / aov.group_stats["count"]).to_numpy() gvar = aov.table.at["Residual", "mean_sq"] / gnobs g1, g2 = np.array(list(itertools.combinations(np.arange(n_groups), 2))).T mn = gmeans[g1] - gmeans[g2] se = np.sqrt(gvar[g1] + gvar[g2]) tval = mn / se df = aov.table.at["Residual", "df"] pval = psturng(np.sqrt(2) * np.abs(tval), n_groups, df) thsd = pd.DataFrame( columns=[ "A", "B", "mean(A)", "mean(B)", "diff", "Std.Err.", "t value", "Pr(>|t|)", ], index=range(n_groups * (n_groups - 1) // 2), ) thsd["A"] = categories[g1] thsd["B"] = categories[g2] thsd["mean(A)"] = gmeans[g1] thsd["mean(B)"] = gmeans[g2] thsd["diff"] = mn thsd["Std.Err."] = se thsd["t value"] = tval thsd["Pr(>|t|)"] = pval return thsd
def get_tukey(df_groups, measure, groups): # Tukey posthoc analysis # See https://jpktd.blogspot.com/2013/03/multiple-comparison-and-tukey-hsd-or_25.html # And https://code.google.com/archive/p/qsturng-py/ # And https://stackoverflow.com/questions/48200699/how-can-i-get-p-values-of-each-group-comparison-when-applying-the-tukey-s-hones # q, res_table, std_pairs, etc can be found from print(dir(result)) which will list all possible calculations df_tukey = df_groups[3][np.isfinite(df_groups[3][measure])] #print(df_tukey) mc = MultiComparison(df_tukey[measure], df_tukey['strain']) #result = pairwise_tukeyhsd(mc.data,mc.groups,0.05) result = mc.tukeyhsd() p = psturng(np.abs(result.meandiffs / result.std_pairs), len(result.groupsunique), result.df_total) df_pairs = pd.DataFrame({ 'group1': [ result._results_table[1][0], result._results_table[2][0], result._results_table[3][0] ], 'group2': [ result._results_table[1][1], result._results_table[2][1], result._results_table[3][1] ], 'p_value': [np.around(p[0], 4), np.around(p[1], 4), np.around(p[2], 4)] }) for index, row in df_pairs.iterrows(): stars = get_stars(row['p_value']) df_pairs.loc[index, 'significance'] = stars return df_pairs
def test_handful_to_known_values(self): cases = [ (0.71499578726111435, 67, 956.70742488392386, 5.0517658443070692), (0.42974234855067672, 16, 723.50261736502318, 3.3303582093701354), (0.94936429359548424, 2, 916.1867328010926, 2.7677975546417244), (0.85357381770725038, 66, 65.67055060832368, 5.5647438108270109), (0.87372108021900929, 74, 626.42369474993632, 5.5355540570701107), (0.53891960564713726, 49, 862.63799438485785, 4.5108645923377146), (0.98818659555664567, 18, 36.269686711464274, 6.0906643750886156), (0.53031994896037626, 50, 265.29558652727917, 4.5179640079726795), (0.7318857887397332, 59, 701.41497552251201, 4.9980139875409915), (0.65332019368982697, 61, 591.01183664195912, 4.8706581766706893), (0.55403221657248558, 77, 907.34156725405194, 4.8786135917984632), (0.30783916857266003, 83, 82.446923487980882, 4.4396401242858294), (0.29321720242415661, 16, 709.64382575553009, 3.0304277540702729), (0.27146478168880306, 31, 590.00594683574172, 3.5870031664477215), (0.67348796958433776, 81, 608.02706111127657, 5.1096199974432936), (0.32774393945968938, 18, 17.706224399250839, 3.2119038163765432), (0.7081637474795982, 72, 443.10678914889695, 5.0990030889410649), (0.33354939276757861, 47, 544.0772192199048, 4.0613352964193279), (0.60412143947363051, 36, 895.83526933271548, 4.381717596850172), (0.88739052300665977, 77, 426.03665511558262, 5.6333929480341309), ] for p, r, v, q in cases: assert_almost_equal(1.0 - p, psturng(q, r, v), 5)
def test_vector(self): "vector input -> vector output" assert_array_almost_equal( np.array([0.10679889, 0.06550009, 0.01730145]), psturng([3.98832389, 4.56835318, 6.26400894], [4, 4, 4], [6, 6, 6]), 5, )
def getOWANOVAmultiComp(data, labels, verbose=False): tlabels = np.concatenate([[labels[j] for _, y in enumerate(x)] for j, x in enumerate(data)]) res = pairwise_tukeyhsd(np.concatenate(data), tlabels) if verbose: print(res.summary()) return psturng(np.abs(res.meandiffs / res.std_pairs), len(res.groupsunique), res.df_total)
def posthoc_turron_by_first_time_tasting(melted, variable): df = melted.copy() df = df[df['variable'] == variable].dropna() df['turron:first_time_tasting'] = df['turron'] + "_" + df['first_time_tasting'] mc = MultiComparison(df['value'], df['turron:first_time_tasting']) res = mc.tukeyhsd() from statsmodels.stats.libqsturng import psturng p_values = psturng(np.abs(res.meandiffs / res.std_pairs), len(res.groupsunique), res.df_total) print(p_values) print(res)
def _games_howell_test(self): combs = list(combinations(np.unique(self.group), 2)) sample_stats = self._group_sample_statistics() means_d = dict(sample_stats['Group Means']) obs_d = dict(sample_stats['Group Observations']) var_d = dict(sample_stats['Group Variance']) group_comps = [] mean_differences = [] degrees_freedom = [] t_values = [] p_values = [] std_err = [] up_conf = [] low_conf = [] for comb in combs: diff = means_d[comb[1]] - means_d[comb[0]] t_val = np.absolute(diff) / np.sqrt((var_d[comb[0]] / obs_d[comb[0]]) + (var_d[comb[1]] / obs_d[comb[1]])) df_num = (var_d[comb[0]] / obs_d[comb[0]] + var_d[comb[1]] / obs_d[comb[1]]) ** 2 df_denom = ((var_d[comb[0]] / obs_d[comb[0]]) ** 2 / (obs_d[comb[0]] - 1) + (var_d[comb[1]] / obs_d[comb[1]]) ** 2 / (obs_d[comb[1]] - 1)) df = df_num / df_denom p_val = psturng(t_val * np.sqrt(2), sample_stats['Number of Groups'], df) se = np.sqrt(0.5 * (var_d[comb[0]] / obs_d[comb[0]] + var_d[comb[1]] / obs_d[comb[1]])) upper_conf = diff + qsturng(1 - self.alpha, sample_stats['Number of Groups'], df) lower_conf = diff - qsturng(1 - self.alpha, sample_stats['Number of Groups'], df) mean_differences.append(diff) degrees_freedom.append(df) t_values.append(t_val) p_values.append(p_val) std_err.append(se) up_conf.append(upper_conf) low_conf.append(lower_conf) group_comps.append(str(comb[0]) + ' : ' + str(comb[1])) result_df = pd.DataFrame({'groups': group_comps, 'mean_difference': mean_differences, 'std_error': std_err, 't_value': t_values, 'p_value': p_values, 'upper_limit': up_conf, 'lower limit': low_conf}) return result_df
def test_1000_random_values(self): n = 1000 ps = np.random.random(n) * (.999 - .1) + .1 rs = np.random.random_integers(2, 100, n) vs = np.random.random(n) * 998. + 2. qs = qsturng(ps, rs, vs) estimates = psturng(qs, rs, vs) actuals = 1. - ps errors = estimates - actuals assert_equal(np.array([]), np.where(errors > 1e-5)[0])
def tukeyTest(data, groups, alpha=0.05): '''Perform pairwise Tukey test for data by groups ''' # pairwise comparisons using Tukey's test, calculating p-values res = pairwise_tukeyhsd(data, groups, alpha) print('Summary of test:\n', res) # print(dir(results))# prints out all attributes of an object pVal = psturng(np.abs(res.meandiffs / res.std_pairs), len(res.groupsunique), res.df_total) print('p values of all pair-wise tests:\n', pVal) return res
def test_vector(self): "vector input -> vector output" assert_array_almost_equal(np.array([0.10679889, 0.06550009, 0.01730145]), psturng([3.98832389, 4.56835318, 6.26400894], [4, 4, 4], [6, 6, 6]), 5)
def test_100_random_values(self): n = 100 ps = np.random.random(n)*(.999 - .1) + .1 rs = np.random.random_integers(2, 100, n) vs = np.random.random(n)*998. + 2. qs = qsturng(ps, rs, vs) estimates = psturng(qs, rs, vs) actuals = 1. - ps errors = estimates - actuals assert_equal(np.array([]), np.where(errors > 1e-5)[0])
def test_qstrung(): rows = [ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 30, 40, 60, 120, 9999] cols = np.arange(2,11) for alpha in [0.01, 0.05]: for k in cols: c1 = get_tukeyQcrit(k, rows, alpha=alpha) c2 = qsturng(1-alpha, k, rows) assert_almost_equal(c1, c2, decimal=2) #roundtrip assert_almost_equal(psturng(qsturng(1-alpha, k, rows), k, rows), alpha, 5)
def test_100_random_values(self, reset_randomstate): n = 100 random_state = np.random.RandomState(12345) ps = random_state.random_sample(n) * (.999 - .1) + .1 rs = random_state.randint(2, 101, n) vs = random_state.random_sample(n) * 998. + 2. qs = qsturng(ps, rs, vs) estimates = psturng(qs, rs, vs) actuals = 1. - ps errors = estimates - actuals assert_equal(np.array([]), np.where(errors > 1e-5)[0])
def tukey_hsd(args, ddof = 1): """ Usage ------ tukey(args, ddof=1.0) Parameters -------- args : pandas dataframe datasets pd.DataFrame({'A':[x11,x12,x13,,,,x1i], 'B':[x21,x22,x23,,,,x2j], 'K':[..................] 'N':[xn1,xn2,xn3,,,,xnk]]) ddof : delta degrees of freedom, default is 1. Returns ------- dict of 'summary', 'p', 'v','var_e' summary : contains mean and var within group, and size of each groups. p : contains dataframe of p-values between each groups. t : contains dataframe of t-values between each groups. var_e : contains df(degrees of freedom of error-variance), and error-variance between groups. Comments ------- Multiple comparison with Tukey-Kramer's range test. This program is a ported python-script from R-script called tukey, originally coded by Dr Shigeyuki Aoki. psturng and qsturng functions were imported from statsmodels. This program was wirtten by Kyoichiro Higashi, ([email protected]) """ ddof = ddof data = args alldata = args.unstack().values groups = len(args.columns) # number of groups n_wig = args.notna().sum(0) # cases within groups phi_e = (n_wig - ddof).sum() # sum of degrees of freedom mean_wig = data.mean() # means of each groups var_wig = data.var() # variance of each groups idx_group = args.columns summary = pd.DataFrame({'size':n_wig, 'mean':mean_wig, 'var':var_wig}, index=idx_group) # statics of each groups var_e = sum((n_wig - ddof) * var_wig) / phi_e # Error variance (Variance within group) t = np.array( [np.array( [ abs(mean_wig[i] - mean_wig[j]) / np.sqrt(var_e * (1. / n_wig[i] + 1. / n_wig[j])) for i in range(groups)] ) for j in range(groups)] ) # t statistics of paired comparison prob = psturng(t*np.sqrt(2.0), groups, phi_e) t = pd.DataFrame(data = t,index = idx_group, columns = idx_group) prob = pd.DataFrame(data = prob,index = idx_group, columns = idx_group) variance = pd.DataFrame(data = [phi_e, var_e], index = ['df','var'], columns = ['error']) return {'summary':summary, 't':t, 'p':prob, 'var_e':variance}
def test_100_random_values(self, reset_randomstate): n = 100 random_state = np.random.RandomState(12345) ps = random_state.random_sample(n)*(.999 - .1) + .1 rs = random_state.randint(2, 101, n) vs = random_state.random_sample(n)*998. + 2. qs = qsturng(ps, rs, vs) estimates = psturng(qs, rs, vs) actuals = 1. - ps errors = estimates - actuals assert_equal(np.array([]), np.where(errors > 1e-5)[0])
def test_qstrung(alpha, k): rows = [ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 30, 40, 60, 120, 9999 ] c1 = get_tukeyQcrit(k, rows, alpha=alpha) c2 = qsturng(1 - alpha, k, rows) assert_almost_equal(c1, c2, decimal=2) # roundtrip assert_almost_equal(psturng(qsturng(1 - alpha, k, rows), k, rows), alpha, 5)
def _group_comparison(self): r""" Constructs a pandas DataFrame containing the test results and group comparisons as found by Tukey's HSD test. Returns ------- groups : array-like pandas DataFrame of group comparison results. """ group_means = npi.group_by(self.design_matrix[:, 0], self.design_matrix[:, 1], np.mean) group_means = [i for _, i in group_means] group_mean_differences = np.array(list(combinations(group_means, 2)))[:, 0] - \ np.array(list(combinations(group_means, 2)))[:, 1] group_sd = npi.group_by(self.design_matrix[:, 0], self.design_matrix[:, 1], std_dev) group_sd = [i for _, i in group_sd] group_names = np.unique(self.design_matrix[:, 0]) groups = pd.DataFrame(np.array(list(combinations(group_names, 2)))) groups['groups'] = groups[0] + ' - ' + groups[1] groups['group means'] = group_means groups['mean difference'] = group_mean_differences groups['std_dev'] = group_sd groups['significant difference'] = np.where( np.abs(groups['mean difference']) >= self.hsd, True, False) groups['upper interval'] = groups[ 'mean difference'] + self.tukey_q_value * np.sqrt( self.mse / 2. * (2. / (self.n / self.k))) groups['lower interval'] = groups[ 'mean difference'] - self.tukey_q_value * np.sqrt( self.mse / 2. * (2. / (self.n / self.k))) q_values = groups['mean difference'] / group_sd groups['p_adjusted'] = psturng(np.absolute(q_values), self.n / self.k, self.dof) del groups[0] del groups[1] return groups
def get_pairwise_comparison_data(df, independent_variables_names, dependent_variables_names, significance_cutoff=0.05, NUM_GROUPS_CUTOFF=15): ''' datasetId independentVariables - list names, must be categorical dependentVariables - list names, must be numerical numBins - number of bins for the independent quantitative variables (if they exist) ''' considered_independent_variable_name = independent_variables_names[0] considered_dependent_variable_name = dependent_variables_names[0] # Only return pairwise comparison data if number of groups < THRESHOLD num_groups = len(get_unique(df[considered_independent_variable_name])) if num_groups > NUM_GROUPS_CUTOFF: return None hsd_result = pairwise_tukeyhsd(df[considered_dependent_variable_name], df[considered_independent_variable_name], alpha=significance_cutoff) hsd_raw_data = hsd_result.summary().data[1:] st_range = np.abs(hsd_result.meandiffs) / hsd_result.std_pairs p_values = psturng(st_range, len(hsd_result.groupsunique), hsd_result.df_total) hsd_headers = [ 'Group 1', 'Group 2', 'Group Mean Difference (2 - 1)', 'Lower Bound', 'Upper Bound', 'p-value', 'Distinct (p < %s)' % significance_cutoff ] hsd_data = [] for i in range(0, len(hsd_raw_data)): if isinstance(p_values, float): p_value = p_values else: p_value = p_values[i] if i < len(p_values) else None hsd_data_row = [ hsd_raw_data[i][0], hsd_raw_data[i][1], hsd_result.meandiffs[i], hsd_result.confint[i][0], hsd_result.confint[i][1], p_value, ( 'True' if (p_value <= significance_cutoff) else 'False' ) ] hsd_data.append(hsd_data_row) return { 'column_headers': hsd_headers, 'rows': hsd_data }
def test_qstrung(): rows = [ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 30, 40, 60, 120, 9999 ] cols = np.arange(2, 11) for alpha in [0.01, 0.05]: for k in cols: c1 = get_tukeyQcrit(k, rows, alpha=alpha) c2 = qsturng(1 - alpha, k, rows) assert_almost_equal(c1, c2, decimal=2) #roundtrip assert_almost_equal(psturng(qsturng(1 - alpha, k, rows), k, rows), alpha, 5)
class Generator: enabled = False display_name = "statsmodel" @staticmethod def init_parser(parser): parser.add_argument('--statsmodel', action="store_true") @classmethod def init_args(cls, arg): cls.enabled = arg.statsmodel @staticmethod def process(case, dop): q, k, nu = case.q, case.k, case.v atol = 10**(-dop) return 1 - psturng(q, k, nu)
def tukey_range_test(*args): """Found in statsmodels as pairwise_tukeyhsd This test compares all possible pairs of means and determines if there are any differences in these pairs. Parameters ---------- args: list or numpy arrays, 1-D The observed measurements for each group, organized into lists or numpy arrays Return ------ results: list A list of lists containing 3 attributes: 1) The groups being compared 2) The Q Statistic 3) p, or the likelihood our observed differences are due to chance """ k = len(args) if k < 2: raise AttributeError("Need at least two groups to perform Tukey Range Test") results = [] mean_i = np.mean(args, axis=1) groups = np.arange(len(args)) n_i = [len(arg) for arg in args] sum_data = np.sum(args, axis=1) square_data = np.power(args, 2) df = sum(n_i) - k sse = _sse(sum_data, square_data, n_i) for group in groups: group = int(group) for next_group in range(group + 1, len(groups)): mean_a, mean_b = mean_i[group], mean_i[next_group] n_a, n_b = n_i[group], n_i[next_group] difference = abs(mean_a - mean_b) std_group = sqrt(sse / df / min(n_a, n_b)) q = difference / std_group p = psturng(q, k, df) results.append(['group {} - group {}'.format(group, next_group), q, p]) return results
def Tukey_p_value(result_from_tukey): #print(result_from_tukey.meandiffs) P_value = psturng( np.abs(result_from_tukey.meandiffs / result_from_tukey.std_pairs), len(result_from_tukey.groupsunique), result_from_tukey.df_total) print(P_value)
def calculate_tukey_posthoc(df, column, type_column='type', verbose=True, write=False, name=None, output_dir=None): """Computes p-values using ANOVA with post-hoc Tukey HSD for a given DataFrame. Estimates p-values for a given DataFrame assuming that the sample type is named as `type_column`. Parameters ---------- df : pandas DataFrame Contains the table of values ans corresponding types or classes. column : str Indicates the column of values. type_column : str Indicates the column of sample kind. verbose : boolean Specifies if the output should be printed into a terminal. write : boolean Specifies if the output should be written into a text file. name : str Indicates the name of the output file. output_dir : str Indicates the output dir where the file will be written. Returns ------- dict : sample typles and p-values The dict of sample types and cooresponding p-values. """ mc = MultiComparison(df[column], df[type_column]) tt = mc.tukeyhsd() st_range = np.abs(tt.meandiffs) / tt.std_pairs fout = None if write and output_dir: if not os.path.exists(output_dir): os.makedirs(output_dir) fout = open(os.path.join(output_dir, name + '.txt'), 'w') print(os.path.join(output_dir, name + '.txt')) if write: print('Tukey post-hoc ({0})'.format(column), end="", file=fout) print(tt, end="", file=fout) print(mc.groupsunique, end="", file=fout) if verbose: print('Tukey post-hoc ({0})' % (column)) print(tt) print(mc.groupsunique) pvals = psturng(st_range, len(tt.groupsunique), tt.df_total) out = {} groups = mc.groupsunique g1idxs, g2idxs = mc.pairindices for g1i, g2i, p in zip(g1idxs, g2idxs, pvals): gname = '{}-{}'.format(groups[g1i], groups[g2i]) out[gname] = p min_item = min(out.iteritems(), key=operator.itemgetter(1)) for grp, p in out.items(): if fout and write: print >> fout, '{}: {}'.format(grp, p) if verbose: print(grp, ': ', p) return out, min_item
def kw_nemenyi(groups, to_compare=None, alpha=0.05, method='tukey'): """ Kruskal-Wallis 1-way ANOVA with Nemenyi's multiple comparison test Arguments: --------------- groups: sequence arrays corresponding to k mutually independent samples from continuous populations to_compare: sequence tuples specifying the indices of pairs of groups to compare, e.g. [(0, 1), (0, 2)] would compare group 0 with 1 & 2. by default, all possible pairwise comparisons between groups are performed. alpha: float family-wise error rate used for correcting for multiple comparisons (see statsmodels.stats.multitest.multipletests for details) method: string the null distribution of the test statistic used to determine the corrected p-values for each pair of groups, can be either "tukey" (studentized range) or "chisq" (Chi-squared). the "chisq" method will correct for tied ranks. Returns: --------------- H: float Kruskal-Wallis H-statistic p_omnibus: float p-value corresponding to the global null hypothesis that the medians of the groups are all equal p_corrected: float array corrected p-values for each pairwise comparison, corresponding to the null hypothesis that the pair of groups has equal medians. note that these are only meaningful if the global null hypothesis is rejected. reject: bool array True for pairs where the null hypothesis can be rejected for the given alpha Reference: --------------- """ # omnibus test (K-W ANOVA) # ------------------------------------------------------------------------- if method is None: method = 'chisq' elif method not in ('tukey', 'chisq'): raise ValueError('method must be either "tukey" or "chisq"') groups = [np.array(gg) for gg in groups] k = len(groups) n = np.array([len(gg) for gg in groups]) if np.any(n < 5): warnings.warn("Sample sizes < 5 are not recommended (K-W test assumes " "a chi square distribution)") allgroups = np.concatenate(groups) N = len(allgroups) ranked = stats.rankdata(allgroups) # correction factor for ties T = stats.tiecorrect(ranked) if T == 0: raise ValueError('All numbers are identical in kruskal') # sum of ranks for each group j = np.insert(np.cumsum(n), 0, 0) R = np.empty(k, dtype=np.float) for ii in range(k): R[ii] = ranked[j[ii]:j[ii + 1]].sum() # the Kruskal-Wallis H-statistic H = (12. / (N * (N + 1.))) * ((R**2.) / n).sum() - 3 * (N + 1) # apply correction factor for ties H /= T df_omnibus = k - 1 p_omnibus = stats.chisqprob(H, df_omnibus) # multiple comparisons # ------------------------------------------------------------------------- # by default we compare every possible pair of groups if to_compare is None: to_compare = tuple(combinations(range(k), 2)) ncomp = len(to_compare) dif = np.empty(ncomp, dtype=np.float) B = np.empty(ncomp, dtype=np.float) Rmean = R / n A = N * (N + 1) / 12. for pp, (ii, jj) in enumerate(to_compare): # absolute difference of mean ranks dif[pp] = np.abs(Rmean[ii] - Rmean[jj]) B[pp] = (1. / n[ii]) + (1. / n[jj]) if method == 'tukey': # p-values obtained from the upper quantiles of the studentized range # distribution qval = dif / np.sqrt(A * B) p_corrected = psturng(qval * np.sqrt(2), k, 1E6) elif method == 'chisq': # p-values obtained from the upper quantiles of the chi-squared # distribution chi2 = (dif**2.) / (A * B) p_corrected = stats.chisqprob(chi2 * T, k - 1) reject = p_corrected <= alpha return H, p_omnibus, p_corrected, reject
def get_group_measures(preds_all, low_ms=None, norm_ms=None, U1=None, U2=None, U3=None, U4=None): predictions = defaultdict(list) maes = defaultdict(list) predictions["All"] = preds_all for user_id, item_id, r, r_, details in preds_all: mae = np.abs(r - r_) maes["All"].append(mae) if low_ms is not None and norm_ms is not None: for user_id, item_id, r, r_, details in preds_all: mae = np.abs(r - r_) if user_id in low_ms: maes["LowMs"].append(mae) predictions["LowMs"].append((user_id, item_id, r, r_, details)) elif user_id in norm_ms: maes["NormMs"].append(mae) predictions["NormMs"].append((user_id, item_id, r, r_, details)) # H0: MAE(NormMs) >= MAE(LowMs) t_statistic, p_val = ttest_ind(maes["NormMs"], maes["LowMs"]) print(p_val) mae_ms_p = p_val / 2.0 print("[t-TEST] t-statistic: %f, p-value: %f" % (t_statistic, mae_ms_p)) if U1 is not None and U2 is not None and U3 is not None and U4 is not None: for user_id, item_id, r, r_, details in preds_all: mae = np.abs(r - r_) if user_id in U1: predictions["U1"].append((user_id, item_id, r, r_, details)) maes["U1"].append(mae) elif user_id in U2: predictions["U2"].append((user_id, item_id, r, r_, details)) maes["U2"].append(mae) elif user_id in U3: predictions["U3"].append((user_id, item_id, r, r_, details)) maes["U3"].append(mae) elif user_id in U4: predictions["U4"].append((user_id, item_id, r, r_, details)) maes["U4"].append(mae) # H0: all means are the same f_val, p_val = f_oneway(maes["U1"], maes["U2"], maes["U3"], maes["U4"]) print("[ANOVA] p-value: %f" % p_val) df = pd.DataFrame(data={"MAE": maes["U1"], "Usergroup": "U1"}) df = df.append(pd.DataFrame(data={"MAE": maes["U2"], "Usergroup": "U2"})) df = df.append(pd.DataFrame(data={"MAE": maes["U3"], "Usergroup": "U3"})) df = df.append(pd.DataFrame(data={"MAE": maes["U4"], "Usergroup": "U4"})) df = df.append(pd.DataFrame(data={"MAE": maes["All"], "Usergroup": "All"})) tukeyhsd_results = pairwise_tukeyhsd(df["MAE"], df["Usergroup"]) print(tukeyhsd_results.summary()) # from https://stackoverflow.com/questions/48200699/how-can-i-get-p-values-of-each-group-comparison-when-applying-the-tukey-s-hones p_values = psturng(np.abs(tukeyhsd_results.meandiffs / tukeyhsd_results.std_pairs), len(tukeyhsd_results.groupsunique), tukeyhsd_results.df_total) print("[TukeyHSD] p-values : " + str(p_values)) results = ResultDict() for group in predictions.keys(): results.add_result(usergroup=group, metric="MAE", value=accuracy.mae(predictions[group], verbose=False)) """results.add_result(usergroup=group, metric="RMSE", value=accuracy.rmse(predictions[group], verbose=False)) results.add_result(usergroup=group, metric="FCP", value=accuracy.fcp(predictions[group], verbose=False))""" """for k in [5, 10, 20]: _, _, f1, mrr, map, ndcg = top_k_measures(predictions[group], k=k) #results.add_result(usergroup=group, metric="P@" + str(k), value=p) #results.add_result(usergroup=group, metric="R@" + str(k), value=r) results.add_result(usergroup=group, metric="F1@" + str(k), value=f1) results.add_result(usergroup=group, metric="MRR@" + str(k), value=mrr) results.add_result(usergroup=group, metric="MAP@" + str(k), value=map) results.add_result(usergroup=group, metric="nDCG@" + str(k), value=ndcg) for k in range(1, 21): p, r, _, _, _, _ = top_k_measures(predictions[group], k=k) results.add_result(usergroup=group, metric="P@" + str(k), value=p) results.add_result(usergroup=group, metric="R@" + str(k), value=r)""" return results
jitter=True, data=final_df[(final_df.Event == 'Cells') | (final_df.Event == treatments[ind])], ax=panel) plt.savefig('respiration.svg') plt.show() #statistical analysis final_df['Slope'] = final_df.Slope.astype(np.float) #fit linear model model = ols('Slope ~ Event + Experiment', data=final_df).fit() print(model.summary()) print('ANOVA analysis') aov_table = sm.stats.anova_lm(model, type=2) print(aov_table) print('Post hoc tukey') mc = MultiComparison(final_df['Slope'], final_df['Event']) mc_results = mc.tukeyhsd() print(mc_results.summary()) p_values = psturng(np.abs(mc_results.meandiffs / mc_results.std_pairs), len(mc_results.groupsunique), mc_results.df_total) print('p_values: ', p_values)
def test_v_equal_one(self): assert_almost_equal(.1, psturng(.2,5,1), 5)
def test_osrt(data: Union[List, np.ndarray, DataFrame], val_col: str = None, group_col: str = None, sort: bool = False) -> Tuple[float, float, int]: '''Hayter's one-sided studentised range test (OSRT) Tests a hypothesis against an ordered alternative for normal data with equal variances [1]_. Parameters ---------- data : Union[List, numpy.ndarray, DataFrame] An array, any object exposing the array interface or a pandas DataFrame with data values. val_col : str = None Name of a DataFrame column that contains dependent variable values (test or response variable). Values should have a non-nominal scale. Must be specified if ``a`` is a pandas DataFrame object. group_col : str = None Name of a DataFrame column that contains independent variable values (grouping or predictor variable). Values should have a nominal scale (categorical). Must be specified if `a` is a pandas DataFrame object. sort : bool = False If True, sort data by block and group columns. Returns ------- Tuple[float, float, int] P value, statistic, and number of degrees of freedom. Notes ----- P values are computed from the Tukey distribution. References ---------- .. [1] Hayter, A.J.(1990) A One-Sided Studentised Range Test for Testing Against a Simple Ordered Alternative, Journal of the American Statistical Association, 85, 778-785. Examples -------- >>> import scikit_posthocs as sp >>> import pandas as pd >>> x = pd.DataFrame({"a": [1,2,3,5,1], "b": [12,31,54,62,12], "c": [10,12,6,74,11]}) >>> x = x.melt(var_name='groups', value_name='values') >>> sp.test_osrt(x, val_col='values', group_col='groups') ''' x, _val_col, _group_col = __convert_to_df(data, val_col, group_col) if not sort: x[_group_col] = Categorical(x[_group_col], categories=x[_group_col].unique(), ordered=True) x.sort_values(by=[_group_col], ascending=True, inplace=True) groups = np.unique(x[_group_col]) x_grouped = x.groupby(_group_col)[_val_col] xi = x_grouped.mean() ni = x_grouped.count() k = groups.size n = len(x.index) df = n - k sigma2 = 0 c = -1 for i in range(k): for j in range(ni.iloc[i]): c += 1 sigma2 += (x[_val_col].iat[c] - xi[i])**2. / df sigma = np.sqrt(sigma2) def compare(i, j): dif = xi.loc[groups[j]] - xi.loc[groups[i]] A = sigma / np.sqrt(2.) * np.sqrt(1. / ni[groups[j]] + 1. / ni[groups[i]]) qval = np.abs(dif) / A return qval vs = np.zeros((k, k), dtype=float) combs = it.combinations(range(k), 2) for i, j in combs: vs[i, j] = compare(i, j) stat = np.max(vs) pval = psturng(stat, k, df) return pval, stat, df
def test_scalar(self): "scalar input -> scalar output" assert_almost_equal(.1, psturng(4.43645545899562, 5, 6), 5)
'P-PG', 'P-TD', 'P-RMSPROP', 'NEAT', 'NEAT-EM-P-PG', 'NEAT-EM-P-TD', 'NEAT-EM-P-RMSPROP' ] group_names = [] for header in headers: group_names += list(itertools.repeat(header, min_length)) mc = MultiComparison(np.asarray(stripped_groups).flatten(), group_names) result = mc.tukeyhsd() from statsmodels.stats.libqsturng import psturng print(result) print(mc.groupsunique) print( psturng(np.abs(result.meandiffs / result.std_pairs), len(result.groupsunique), result.df_total)) def mean_confidence_interval(data, confidence=0.95): a = 1.0 * np.array(data) n = len(a) m, se = np.mean(a), stats.sem(a) h = se * stats.t._ppf((1 + confidence) / 2., n - 1) return h count = 0 for stripped_group in stripped_groups: print(headers[count]) print(np.mean(stripped_group), mean_confidence_interval(stripped_group)) count += 1
def test_v_equal_one(self): assert_almost_equal(.1, psturng(.2, 5, 1), 5)
def test_scalar(self): "scalar input -> scalar output" assert_almost_equal(.1, psturng(4.43645545899562,5,6), 5)