def _pairwise_ttests(self, dv=None, between=None, within=None, subject=None, parametric=True, alpha=.05, tail='two-sided', padjust='none', effsize='hedges', return_desc=False, export_filename=None): """Post-hoc tests.""" posthoc = pairwise_ttests(data=self, dv=dv, between=between, within=within, subject=subject, parametric=parametric, alpha=alpha, tail=tail, padjust=padjust, effsize=effsize, return_desc=return_desc, export_filename=export_filename) return posthoc
def test_pairwise_ttests(self): """Test function pairwise_ttests. Tested against the pairwise.t.test R function.""" df = read_dataset('mixed_anova.csv') # Within + Between + Within * Between pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, alpha=.01) pairwise_ttests(dv='Scores', within=['Time'], between=['Group'], subject='Subject', data=df, padjust='fdr_bh', return_desc=True) # Simple within # In R: # >>> pairwise.t.test(df$Scores, df$Time, pool.sd = FALSE, # ... p.adjust.method = 'holm', paired = TRUE) pt = pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, return_desc=True, padjust='holm') np.testing.assert_array_equal(pt.loc[:, 'p-corr'].round(3), [0.174, 0.024, 0.310]) np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.087, 0.008, 0.310]) pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, parametric=False, return_desc=True) # Simple between # In R: # >>> pairwise.t.test(df$Scores, df$Group, pool.sd = FALSE) pt = pairwise_ttests(dv='Scores', between='Group', data=df).round(3) assert pt.loc[0, 'p-unc'] == 0.023 pairwise_ttests(dv='Scores', between='Group', data=df, padjust='bonf', tail='one-sided', effsize='cohen', parametric=False, export_filename='test_export.csv') # Two between factors pairwise_ttests(dv='Scores', between=['Time', 'Group'], data=df, padjust='holm') pairwise_ttests(dv='Scores', between=['Time', 'Group'], data=df, padjust='holm', parametric=False) # Two within subject factors pairwise_ttests(dv='Scores', within=['Group', 'Time'], subject='Subject', data=df, padjust='bonf') pairwise_ttests(dv='Scores', within=['Group', 'Time'], subject='Subject', data=df, padjust='bonf', parametric=False) # Wrong tail argument with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df, tail='wrong') # Wrong alpha argument with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df, alpha='.05') # Both multiple between and multiple within with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between=['Time', 'Group'], within=['Time', 'Group'], subject='Subject', data=df) # Missing values df.iloc[[10, 15], 0] = np.nan pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df) # Wrong input argument df['Group'] = 'Control' with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df) # Two within factors from other datasets and with NaN values df2 = read_dataset('rm_anova') pairwise_ttests(dv='DesireToKill', within=['Disgustingness', 'Frighteningness'], subject='Subject', padjust='holm', data=df2) # Compare with JASP tail / parametric argument df = read_dataset('pairwise_ttests') # 1. Within # 1.1 Parametric # 1.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.907, 0.941, 0.405]) assert all(pt.loc[:, 'BF10'].astype(float) < 1) # 1.1.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.595]) assert sum(pt.loc[:, 'BF10'].astype(float) > 1) == 2 # 1.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.405]) # 1.2 Non-parametric # 1.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.910, 0.951, 0.482]) # 1.2.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.554]) # 1.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.482]) # Compare the RBC value for wilcoxon from pingouin.nonparametric import wilcoxon x = df[df['Drug'] == 'A']['Scores'].values y = df[df['Drug'] == 'B']['Scores'].values assert -0.6 < wilcoxon(x, y).at['Wilcoxon', 'RBC'] < -0.4 x = df[df['Drug'] == 'B']['Scores'].values y = df[df['Drug'] == 'C']['Scores'].values assert wilcoxon(x, y).at['Wilcoxon', 'RBC'].round(3) == 0.030 # 2. Between # 2.1 Parametric # 2.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.1.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.932 assert float(pt.loc[0, 'BF10']) < 1 # 2.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.2 Non-parametric # 2.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # 2.2.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.901 # 2.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # Compare the RBC value for MWU from pingouin.nonparametric import mwu x = df[df['Gender'] == 'M']['Scores'].values y = df[df['Gender'] == 'F']['Scores'].values assert abs(mwu(x, y).at['MWU', 'RBC']) == 0.252
def test_pairwise_ttests(self): """Test function pairwise_ttests. Tested against the pairwise.t.test R function, as well as JASP and JAMOVI. Notes: 1) JAMOVI by default pool the error term for the within-subject factor in mixed design. Pingouin does not pool the error term, which is the same behavior as JASP. 2) JASP does not return the uncorrected p-values, therefore only the corrected p-values are compared. 3) JASP does not calculate the Bayes Factor for the interaction terms. For mixed design and two-way design, in JASP, the Bayes Factor seems to be calculated without aggregating over repeated measurements. 4) For factorial between-subject contrasts, both JASP and JAMOVI pool the error term. This option is not yet implemented in Pingouin. Therefore, one cannot directly validate the T and p-values. """ df = read_dataset('mixed_anova.csv') # Simple and mixed design df_sort = df.sort_values('Time') # Same but random order of subject # df_sort = df.sample(frac=1) df_unb = read_dataset('mixed_anova_unbalanced') df_rm2 = read_dataset('rm_anova2') # 2-way rm design df_aov2 = read_dataset('anova2') # 2-way factorial design # ------------------------------------------------------------------- # Simple within: EASY! # ------------------------------------------------------------------- # In R: # >>> pairwise.t.test(df$Scores, df$Time, pool.sd = FALSE, # ... p.adjust.method = 'holm', paired = TRUE) pt = pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, return_desc=True, padjust='holm') np.testing.assert_array_equal(pt.loc[:, 'p-corr'].round(3), [0.174, 0.024, 0.310]) np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.087, 0.008, 0.310]) pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, parametric=False, return_desc=True) # Same after random ordering of subject (issue 151) pt_sort = pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df_sort, return_desc=True, padjust='holm') assert pt_sort.equals(pt) # ------------------------------------------------------------------- # Simple between: EASY! # ------------------------------------------------------------------- # In R: >>> pairwise.t.test(df$Scores, df$Group, pool.sd = FALSE) pt = pairwise_ttests(dv='Scores', between='Group', data=df).round(3) assert pt.loc[0, 'p-unc'] == 0.023 pairwise_ttests(dv='Scores', between='Group', data=df, padjust='bonf', tail='one-sided', effsize='cohen', parametric=False) # Same after random ordering of subject (issue 151) pt_sort = pairwise_ttests(dv='Scores', between='Group', data=df_sort).round(3) assert pt_sort.equals(pt) # ------------------------------------------------------------------- # Mixed design: Within + Between + Within * Between # ------------------------------------------------------------------- # .Balanced data # ..With marginal means pt = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, padjust='holm', interaction=False) # ...Within main effect: OK with JASP assert np.array_equal(pt['Paired'], [True, True, True, False]) assert np.array_equal(pt.loc[:2, 'p-corr'].round(3), [0.174, 0.024, 0.310]) assert np.array_equal(pt.loc[:2, 'BF10'].astype(float), [0.582, 4.232, 0.232]) # ..Between main effect: T and p-values OK with JASP # but BF10 is only similar when marginal=False (see note in the # 2-way RM test below). assert pt.loc[3, 'T'].round(3) == -2.248 assert pt.loc[3, 'p-unc'].round(3) == 0.028 # ..Interaction: slightly different because JASP pool the error term # across the between-subject groups. JASP does not compute the BF10 # for the interaction. pt = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, padjust='holm', interaction=True).round(5) # Same after random ordering of subject (issue 151) pt_sort = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df_sort, padjust='holm', interaction=True).round(5) assert pt_sort.equals(pt) # ..Changing the order of the model with ``within_first=False``. # output model is now between + within + between * within. # https://github.com/raphaelvallat/pingouin/issues/102 pt = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, padjust='holm', within_first=False) # This should be equivalent to manually filtering dataframe to keep # only one level at a time of the between factor and then running # a within-subject pairwise T-tests. pt_con = pairwise_ttests(dv='Scores', within='Time', subject='Subject', padjust='holm', data=df[df['Group'] == 'Control']) pt_med = pairwise_ttests(dv='Scores', within='Time', subject='Subject', padjust='holm', data=df[df['Group'] == 'Meditation']) pt_merged = pt_con.append(pt_med) # T, dof and p-values should be equal assert np.array_equal(pt_merged['T'], pt['T'].iloc[4:]) assert np.array_equal(pt_merged['dof'], pt['dof'].iloc[4:]) assert np.array_equal(pt_merged['p-unc'], pt['p-unc'].iloc[4:]) # However adjusted p-values are not equal because they are calculated # separately on each dataframe. assert not np.array_equal(pt_merged['p-corr'], pt['p-corr'].iloc[4:]) # I also manually checked the previous lines using parametric=False and # one-sided test. # Other options pairwise_ttests(dv='Scores', within=['Time'], between=['Group'], subject='Subject', data=df, padjust='fdr_bh', alpha=.01, return_desc=True, parametric=False) # .Unbalanced data # ..With marginal means pt1 = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df_unb, padjust='bonf') # ...Within main effect: OK with JASP assert np.array_equal(pt1.loc[:5, 'T'].round(3), [-0.777, -1.344, -2.039, -0.814, -1.492, -0.627]) assert np.array_equal(pt1.loc[:5, 'p-corr'].round(3), [1., 1., 0.313, 1., 0.889, 1.]) assert np.array_equal(pt1.loc[:5, 'BF10'].astype(float), [0.273, 0.463, 1.221, 0.280, 0.554, 0.248]) # ...Between main effect: slightly different from JASP (why?) # True with or without the Welch correction... assert (pt1.loc[6:8, 'p-corr'] > 0.20).all() # ...Interaction: slightly different because JASP pool the error term # across the between-subject groups. # Below the interaction JASP bonferroni-correct p-values, which are # more conservative because JASP perform all possible pairwise tests # jasp_pbonf = [1., 1., 1., 1., 1., 1., 1., 0.886, 1., 1., 1., 1.] assert (pt1.loc[9:, 'p-corr'] > 0.05).all() # Check that the Welch corection is applied by default assert not pt1['dof'].apply(lambda x: x.is_integer()).all() # Same after random ordering of subject (issue 151) pt_sort = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df_unb.sample(frac=1, replace=False), padjust='bonf') assert pt_sort.round(5).equals(pt1.round(5)) # ..No marginal means pt2 = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df_unb, padjust='bonf', marginal=False) # This only impacts the between-subject contrast np.array_equal( (pt1['T'] == pt2['T']).astype(int), [1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) assert (pt1.loc[6:8, 'dof'] < pt2.loc[6:8, 'dof']).all() # Without the Welch correction, check that all the DF are integer pt3 = pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df_unb, correction=False) assert pt3['dof'].apply(lambda x: x.is_integer()).all() # ------------------------------------------------------------------- # Two between factors (FACTORIAL) # ------------------------------------------------------------------- pt = df_aov2.pairwise_ttests(dv='Yield', between=['Blend', 'Crop'], padjust='holm').round(3) # The T and p-values are close but not exactly the same as JASP / # JAMOVI, because they both pool the error term. # The dof are not available in JASP, but in JAMOVI they are 18 # everywhere, which I'm not sure to understand why... assert np.array_equal(pt.loc[:3, 'p-unc'] < 0.05, [False, False, False, True]) # However, the Bayes Factor of the simple main effects are the same...! np.array_equal(pt.loc[:3, 'BF10'].astype(float), [0.374, 0.533, 0.711, 2.287]) # Using the Welch method (all df should be non-integer) pt_c = df_aov2.pairwise_ttests(dv='Yield', between=['Blend', 'Crop'], padjust='holm', correction=True) assert not pt_c['dof'].apply(lambda x: x.is_integer()).any() # The ``marginal`` option has no impact here. assert pt.equals( df_aov2.pairwise_ttests(dv='Yield', between=['Blend', 'Crop'], padjust='holm', marginal=True).round(3)) # ------------------------------------------------------------------- # Two within subject factors # ------------------------------------------------------------------- # .Marginal = True ptw1 = pairwise_ttests(data=df_rm2, dv='Performance', within=['Time', 'Metric'], subject='Subject', padjust='bonf', marginal=True).round(3) # Compare the T values of the simple main effect against JASP # Note that the T-values of the interaction are slightly different # because JASP pool the error term. assert np.array_equal(ptw1.loc[0:3, 'T'], [5.818, 1.559, 7.714, 5.110]) # Random sorting of the dataframe (issue 151) pt_sort = pairwise_ttests(data=df_rm2.sample(frac=1), dv='Performance', within=['Time', 'Metric'], subject='Subject', padjust='bonf', marginal=True).round(3) assert pt_sort.equals(ptw1) # Non-parametric (mostly for code coverage) pairwise_ttests(data=df_rm2, dv='Performance', within=['Time', 'Metric'], subject='Subject', parametric=False) # ------------------------------------------------------------------- # ERRORS # ------------------------------------------------------------------- # Both multiple between and multiple within with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between=['Time', 'Group'], within=['Time', 'Group'], subject='Subject', data=df) # Wrong input argument df['Group'] = 'Control' with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df) # ------------------------------------------------------------------- # Missing values in repeated measurements # ------------------------------------------------------------------- # 1. Parametric df = read_dataset('pairwise_ttests_missing') st = pairwise_ttests(dv='Value', within='Condition', subject='Subject', data=df, nan_policy='listwise') np.testing.assert_array_equal(st['dof'].to_numpy(), [7, 7, 7]) st2 = pairwise_ttests(dv='Value', within='Condition', data=df, subject='Subject', nan_policy='pairwise') np.testing.assert_array_equal(st2['dof'].to_numpy(), [8, 7, 8]) # 2. Non-parametric st = pairwise_ttests(dv='Value', within='Condition', subject='Subject', data=df, parametric=False, nan_policy='listwise') np.testing.assert_array_equal(st['W-val'].to_numpy(), [9, 3, 12]) st2 = pairwise_ttests(dv='Value', within='Condition', data=df, subject='Subject', nan_policy='pairwise', parametric=False) # Tested against a simple for loop on combinations np.testing.assert_array_equal(st2['W-val'].to_numpy(), [9, 3, 21]) with pytest.raises(ValueError): # Unbalanced design in repeated measurements df_unbalanced = df.iloc[1:, :].copy() pairwise_ttests(data=df_unbalanced, dv='Value', within='Condition', subject='Subject') # Two within factors from other datasets and with NaN values df2 = read_dataset('rm_anova') pairwise_ttests(dv='DesireToKill', within=['Disgustingness', 'Frighteningness'], subject='Subject', padjust='holm', data=df2) # ------------------------------------------------------------------- # Test tail / parametric argument (compare with JASP) # ------------------------------------------------------------------- df = read_dataset('pairwise_ttests') # 1. Within # 1.1 Parametric # 1.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.907, 0.941, 0.405]) assert all(pt.loc[:, 'BF10'].astype(float) < 1) # 1.1.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.595]) assert sum(pt.loc[:, 'BF10'].astype(float) > 1) == 2 # 1.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.405]) # 1.2 Non-parametric # 1.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.910, 0.951, 0.483]) # 1.2.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.551]) # 1.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.483]) # Compare the RBC value for wilcoxon from pingouin.nonparametric import wilcoxon x = df[df['Drug'] == 'A']['Scores'].to_numpy() y = df[df['Drug'] == 'B']['Scores'].to_numpy() assert -0.6 < wilcoxon(x, y).at['Wilcoxon', 'RBC'] < -0.4 x = df[df['Drug'] == 'B']['Scores'].to_numpy() y = df[df['Drug'] == 'C']['Scores'].to_numpy() assert wilcoxon(x, y).at['Wilcoxon', 'RBC'].round(3) == 0.030 # 2. Between # 2.1 Parametric # 2.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.932 assert float(pt.loc[0, 'BF10']) < 1 # 2.1.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.2 Non-parametric # 2.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.901 # 2.2.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # 2.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # Compare the RBC value for MWU from pingouin.nonparametric import mwu x = df[df['Gender'] == 'M']['Scores'].to_numpy() y = df[df['Gender'] == 'F']['Scores'].to_numpy() assert round(abs(mwu(x, y).at['MWU', 'RBC']), 3) == 0.252
def test_pairwise_ttests(self): """Test function pairwise_ttests""" # Within + Between + Within * Between pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, alpha=.01) pairwise_ttests(dv='Scores', within=['Time'], between=['Group'], subject='Subject', data=df, padjust='fdr_bh', return_desc=True) # Simple within pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, return_desc=True) # Simple between pairwise_ttests(dv='Scores', between='Group', data=df, padjust='bonf', tail='one-sided', effsize='cohen', export_filename='test_export.csv') # Two between factors pairwise_ttests(dv='Scores', between=['Time', 'Group'], data=df, padjust='holm') # Two within subject factors pairwise_ttests(dv='Scores', within=['Group', 'Time'], subject='Subject', data=df, padjust='bonf') # Wrong tail argument with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df, tail='wrong') # Wrong alpha argument with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df, alpha='.05') # Both multiple between and multiple within with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between=['Time', 'Group'], within=['Time', 'Group'], subject='Subject', data=df) # Missing values df.iloc[[10, 15], 0] = np.nan pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df) # Wrong input argument df['Group'] = 'Control' with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df) # Two within factors from other datasets and with NaN values df2 = read_dataset('rm_anova') pairwise_ttests(dv='DesireToKill', within=['Disgustingness', 'Frighteningness'], subject='Subject', padjust='holm', data=df2)
def test_pairwise_ttests(self): """Test function pairwise_ttests. Tested against the pairwise.t.test R function.""" df = read_dataset('mixed_anova.csv') # Within + Between + Within * Between pairwise_ttests(dv='Scores', within='Time', between='Group', subject='Subject', data=df, alpha=.01) pairwise_ttests(dv='Scores', within=['Time'], between=['Group'], subject='Subject', data=df, padjust='fdr_bh', return_desc=True) # Simple within # In R: # >>> pairwise.t.test(df$Scores, df$Time, pool.sd = FALSE, # ... p.adjust.method = 'holm', paired = TRUE) pt = pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, return_desc=True, padjust='holm') np.testing.assert_array_equal(pt.loc[:, 'p-corr'].round(3), [0.174, 0.024, 0.310]) np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.087, 0.008, 0.310]) pairwise_ttests(dv='Scores', within='Time', subject='Subject', data=df, parametric=False, return_desc=True) # Simple between # In R: # >>> pairwise.t.test(df$Scores, df$Group, pool.sd = FALSE) pt = pairwise_ttests(dv='Scores', between='Group', data=df).round(3) assert pt.loc[0, 'p-unc'] == 0.023 pairwise_ttests(dv='Scores', between='Group', data=df, padjust='bonf', tail='one-sided', effsize='cohen', parametric=False, export_filename='test_export.csv') # Two between factors pt = pairwise_ttests(dv='Scores', between=['Time', 'Group'], data=df, padjust='holm').round(3) pairwise_ttests(dv='Scores', between=['Time', 'Group'], data=df, padjust='holm', parametric=False) # .. with no interaction pt_no_inter = df.pairwise_ttests(dv='Scores', between=['Time', 'Group'], interaction=False, padjust='holm').round(3) assert pt.drop(columns=['Time']).iloc[0:4, :].equals(pt_no_inter) # Two within subject factors ptw = pairwise_ttests(data=df, dv='Scores', within=['Group', 'Time'], subject='Subject', padjust='bonf', parametric=False).round(3) ptw_no_inter = df.pairwise_ttests(dv='Scores', within=['Group', 'Time'], subject='Subject', padjust='bonf', interaction=False, parametric=False).round(3) assert ptw.drop(columns=['Group']).iloc[0:4, :].equals(ptw_no_inter) # Both multiple between and multiple within with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between=['Time', 'Group'], within=['Time', 'Group'], subject='Subject', data=df) # Wrong input argument df['Group'] = 'Control' with pytest.raises(ValueError): pairwise_ttests(dv='Scores', between='Group', data=df) # Missing values in repeated measurements # 1. Parametric df = read_dataset('pairwise_ttests_missing') st = pairwise_ttests(dv='Value', within='Condition', subject='Subject', data=df, nan_policy='listwise') np.testing.assert_array_equal(st['dof'].values, [7, 7, 7]) st2 = pairwise_ttests(dv='Value', within='Condition', data=df, subject='Subject', nan_policy='pairwise') np.testing.assert_array_equal(st2['dof'].values, [8, 7, 8]) # 2. Non-parametric st = pairwise_ttests(dv='Value', within='Condition', subject='Subject', data=df, parametric=False, nan_policy='listwise') np.testing.assert_array_equal(st['W-val'].values, [9, 3, 12]) st2 = pairwise_ttests(dv='Value', within='Condition', data=df, subject='Subject', nan_policy='pairwise', parametric=False) # Tested against a simple for loop on combinations np.testing.assert_array_equal(st2['W-val'].values, [9, 3, 21]) with pytest.raises(ValueError): # Unbalanced design in repeated measurements df_unbalanced = df.iloc[1:, :].copy() pairwise_ttests(data=df_unbalanced, dv='Value', within='Condition', subject='Subject') # Two within factors from other datasets and with NaN values df2 = read_dataset('rm_anova') pairwise_ttests(dv='DesireToKill', within=['Disgustingness', 'Frighteningness'], subject='Subject', padjust='holm', data=df2) # Compare with JASP tail / parametric argument df = read_dataset('pairwise_ttests') # 1. Within # 1.1 Parametric # 1.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.907, 0.941, 0.405]) assert all(pt.loc[:, 'BF10'].astype(float) < 1) # 1.1.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.595]) assert sum(pt.loc[:, 'BF10'].astype(float) > 1) == 2 # 1.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.093, 0.059, 0.405]) # 1.2 Non-parametric # 1.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='greater') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.910, 0.951, 0.482]) # 1.2.2 Tail is less pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='less') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.554]) # 1.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', within='Drug', subject='Subject', parametric=False, data=df, tail='one-sided') np.testing.assert_array_equal(pt.loc[:, 'p-unc'].round(3), [0.108, 0.060, 0.482]) # Compare the RBC value for wilcoxon from pingouin.nonparametric import wilcoxon x = df[df['Drug'] == 'A']['Scores'].values y = df[df['Drug'] == 'B']['Scores'].values assert -0.6 < wilcoxon(x, y).at['Wilcoxon', 'RBC'] < -0.4 x = df[df['Drug'] == 'B']['Scores'].values y = df[df['Drug'] == 'C']['Scores'].values assert wilcoxon(x, y).at['Wilcoxon', 'RBC'].round(3) == 0.030 # 2. Between # 2.1 Parametric # 2.1.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.1.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.932 assert float(pt.loc[0, 'BF10']) < 1 # 2.1.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.068 assert float(pt.loc[0, 'BF10']) > 1 # 2.2 Non-parametric # 2.2.1 Tail is greater pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='greater') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # 2.2.2 Tail is less pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='less') assert pt.loc[0, 'p-unc'].round(3) == 0.901 # 2.2.3 Tail is one-sided: smallest p-value pt = pairwise_ttests(dv='Scores', between='Gender', parametric=False, data=df, tail='one-sided') assert pt.loc[0, 'p-unc'].round(3) == 0.105 # Compare the RBC value for MWU from pingouin.nonparametric import mwu x = df[df['Gender'] == 'M']['Scores'].values y = df[df['Gender'] == 'F']['Scores'].values assert abs(mwu(x, y).at['MWU', 'RBC']) == 0.252