def test_multicomp(self): """Test function multicomp""" reject, pvals_corr = multicomp(pvals, method='fdr_bh') reject, pvals_corr = multicomp(pvals, method='fdr_by') reject, pvals_corr = multicomp(pvals, method='h') reject, pvals_corr = multicomp(pvals, method='b') reject, pvals_corr = multicomp(pvals, method='sidak') reject, pvals_corr = multicomp(pvals, method='none') assert_array_equal(pvals, pvals_corr) reject, pvals_corr = multicomp(pvals2, method='holm') # Wrong arguments with pytest.raises(ValueError): reject, pvals_corr = multicomp(pvals, method='wrong')
def pairwise_corr(data, columns=None, tail='two-sided', method='pearson', padjust='none', export_filename=None): '''Pairwise correlations between columns of a pandas dataframe. Parameters ---------- data : pandas DataFrame DataFrame columns : list or str Column names in data :: '["a", "b", "c"]' : combination between columns a, b, and c '["a"]' : product between a and all the other numeric columns '[["a"], ["b", "c"]]' : product between ["a"] and ["b", "c"] '[["a", "d"], ["b", "c"]]' : product between ["a", "d"] and ["b", "c"] '[["a", "d"], None]' : product between ["a", "d"] and all other columns Note that if column is not specified, then the function will return the pairwise correlation between the combination of all the numeric columns in data. See the examples section for more details on this. tail : string Indicates whether to return the 'two-sided' or 'one-sided' p-values method : string Specify which method to use for the computation of the correlation coefficient. Available methods are :: 'pearson' : Pearson product-moment correlation 'spearman' : Spearman rank-order correlation 'kendall' : Kendall’s tau (ordinal data) 'percbend' : percentage bend correlation (robust) 'shepherd' : Shepherd's pi correlation (robust Spearman) padjust : string Method used for testing and adjustment of pvalues. Available methods are :: 'none' : no correction 'bonferroni' : one-step Bonferroni correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction export_filename : string Filename (without extension) for the output file. If None, do not export the table. By default, the file will be created in the current python console directory. To change that, specify the filename with full path. Returns ------- stats : DataFrame Stats summary :: 'X' : Name(s) of first columns 'Y' : Name(s) of second columns 'method' : method used to compute the correlation 'tail' : indicates whether the p-values are one-sided or two-sided 'n' : Sample size (after NaN removal) 'r' : Correlation coefficients 'CI95' : 95% parametric confidence intervals 'r2' : R-squared values 'adj_r2' : Adjusted R-squared values 'z' : Standardized correlation coefficients 'p-unc' : uncorrected one or two tailed p-values 'p-corr' : corrected one or two tailed p-values 'p-adjust' : Correction method Notes ----- Please refer to the `pingouin.corr()` function for a description of the different methods. NaN are automatically removed from the data. This function is more flexible and gives a much more detailed output than the `pandas.DataFrame.corr()` method (i.e. p-values, confidence interval, Bayes Factor..). This comes however at an increased computational cost. While this should not be discernible for dataframe with less than 10,000 rows and/or less than 20 columns, this function can be slow for very large dataset. For speed purpose, the Bayes Factor is only computed when the sample size is less than 1000 (and method='pearson'). Examples -------- 1. One-tailed spearman correlation corrected for multiple comparisons >>> from pingouin.datasets import read_dataset >>> from pingouin import pairwise_corr >>> data = read_dataset('pairwise_corr').iloc[:, 1:] >>> stats = pairwise_corr(data, method='spearman', tail='two-sided', >>> padjust='bonf') >>> stats 2. Robust two-sided correlation with uncorrected p-values >>> pairwise_corr(data, columns=['Openness', 'Extraversion', >>> 'Neuroticism'], method='percbend') 3. Export the results to a .csv file >>> pairwise_corr(data, export_filename='pairwise_corr.csv') 4. One-versus-others pairwise correlations >>> pairwise_corr(data, columns=['Neuroticism']) 5. Pairwise correlations between two lists of columns (cartesian product) >>> pairwise_corr(data, columns=[['Neuroticism', 'Extraversion'], >>> ['Openness', 'Agreeableness']) ''' from pingouin.correlation import corr if tail not in ['one-sided', 'two-sided']: raise ValueError('Tail not recognized') # Keep only numeric columns data = data._get_numeric_data() keys = data.keys().tolist() # Initialize empty DataFrame stats = pd.DataFrame() # First ensure that columns is a list if isinstance(columns, str): columns = [columns] # Then define combinations / products between columns if columns is None: # Case A: column is not defined --> corr between all numeric columns combs = list(combinations(keys, 2)) else: # Case B: column is specified if isinstance(columns[0], list): group1 = [e for e in columns[0] if e in keys] # Assert that column is two-dimensional if len(columns) == 1: columns.append(None) if isinstance(columns[1], list) and len(columns[1]): # B1: [['a', 'b'], ['c', 'd']] group2 = [e for e in columns[1] if e in keys] else: # B2: [['a', 'b']], [['a', 'b'], None] or [['a', 'b'], 'all'] group2 = [e for e in keys if e not in group1] combs = list(product(group1, group2)) else: # Column is a simple list if len(columns) == 1: # Case B3: one-versus-all, e.g. ['a'] or 'a' others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # Combinations between all specified columns ['a', 'b', 'c'] # Make sure that we keep numeric columns columns = np.intersect1d(keys, columns) if len(columns) == 1: # If only one-column is left, equivalent to ['a'] others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # combinations between ['a', 'b', 'c'] combs = list(combinations(columns, 2)) # Assert that all columns do exist in DataFrame # If you see this error, check for column name errors in `columns=[]` for comb in combs: assert comb[0] in keys assert comb[1] in keys # Initialize vectors for comb in combs: col1, col2 = comb # Avoid errors when one of the two columns has only one unique value if data[col1].unique().size == 1 or data[col2].unique().size == 1: continue cor_st = corr(data[col1].values, data[col2].values, tail=tail, method=method).reset_index(drop=True) stats = stats.append( { 'X': col1, 'Y': col2, 'method': method, 'tail': tail, 'n': cor_st['n'][0], 'r': cor_st['r'][0], 'CI95%': cor_st['CI95%'][0], 'r2': cor_st['r2'][0], 'adj_r2': cor_st['adj_r2'][0], 'p-unc': cor_st['p-val'][0], 'BF10': cor_st['BF10'][0] if 'BF10' in cor_st.keys() else np.nan, 'power': cor_st['power'][0] }, ignore_index=True) # Multiple comparisons padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': reject, stats['p-corr'] = multicomp(stats['p-unc'].values, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None # Standardize correlation coefficients (Fisher z-transformation) stats['z'] = np.arctanh(stats['r'].values) # Round values for c in ['r', 'r2', 'adj_r2', 'z']: stats[c] = stats[c].round(3) col_order = [ 'X', 'Y', 'method', 'tail', 'n', 'r', 'CI95%', 'r2', 'adj_r2', 'z', 'p-unc', 'p-corr', 'p-adjust', 'BF10', 'power' ] # Convert n to int stats['n'] = stats['n'].astype(int) stats = stats.reindex(columns=col_order) stats.dropna(how='all', axis=1, inplace=True) if export_filename is not None: _export_table(stats, export_filename) return stats
def pairwise_ttests(dv=None, between=None, within=None, subject=None, data=None, alpha=.05, tail='two-sided', padjust='none', effsize='hedges', return_desc=False, export_filename=None): '''Pairwise T-tests. Parameters ---------- dv : string Name of column containing the dependant variable. between : string or list with 2 elements Name of column(s) containing the between factor(s). within : string or list with 2 elements Name of column(s) containing the within factor(s). subject : string Name of column containing the subject identifier. Compulsory for contrast including a within-subject factor. data : pandas DataFrame DataFrame alpha : float Significance level tail : string Indicates whether to return the 'two-sided' or 'one-sided' p-values padjust : string Method used for testing and adjustment of pvalues. Available methods are :: 'none' : no correction 'bonferroni' : one-step Bonferroni correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction effsize : string or None Effect size type. Available methods are :: 'none' : no effect size 'cohen' : Unbiased Cohen d 'hedges' : Hedges g 'glass': Glass delta 'eta-square' : Eta-square 'odds-ratio' : Odds ratio 'AUC' : Area Under the Curve return_desc : boolean If True, append group means and std to the output dataframe export_filename : string Filename (without extension) for the output file. If None, do not export the table. By default, the file will be created in the current python console directory. To change that, specify the filename with full path. Returns ------- stats : DataFrame Stats summary :: 'A' : Name of first measurement 'B' : Name of second measurement 'Paired' : indicates whether the two measurements are paired or not 'Tail' : indicate whether the p-values are one-sided or two-sided 'T' : T-values 'p-unc' : Uncorrected p-values 'p-corr' : Corrected p-values 'p-adjust' : p-values correction method 'BF10' : Bayes Factor 'efsize' : effect sizes 'eftype' : type of effect size Notes ----- If between or within is a list (e.g. ['col1', 'col2']), the function returns 1) the pairwise T-tests between each values of the first column, 2) the pairwise T-tests between each values of the second column and 3) the interaction between col1 and col2. The interaction is dependent of the order of the list, so ['col1', 'col2'] will not yield the same results as ['col2', 'col1']. In other words, if between is a list with two elements, the output model is between1 + between2 + between1 * between2. Similarly, if within is a list with two elements, the output model is within1 + within2 + within1 * within2. If both between and within are specified, the function return within + between + within * between. Examples -------- 1. One between-factor >>> from pingouin import pairwise_ttests >>> from pingouin.datasets import read_dataset >>> df = read_dataset('mixed_anova.csv') >>> post_hocs = pairwise_ttests(dv='Scores', between='Group', data=df) >>> print(post_hocs) 2. One within-factor >>> post_hocs = pairwise_ttests(dv='Scores', within='Time', >>> subject='Subject', data=df) >>> print(post_hocs) 3. Within + Between + Within * Between with corrected p-values >>> post_hocs = pairwise_ttests(dv='Scores', within='Time', >>> subject='Subject', between='Group', >>> padjust='bonf', data=df) >>> print(post_hocs) 3. Between1 + Between2 + Between1 * Between2 >>> pairwise_ttests(dv='Scores', between=['Group', 'Time'], data=df) ''' from pingouin.parametric import ttest # Safety checks _check_dataframe(dv=dv, between=between, within=within, subject=subject, effects='all', data=data) if tail not in ['one-sided', 'two-sided']: raise ValueError('Tail not recognized') if not isinstance(alpha, float): raise ValueError('Alpha must be float') # Check if we have multiple between or within factors multiple_between = False multiple_within = False contrast = None if isinstance(between, list): if len(between) > 1: multiple_between = True contrast = 'multiple_between' assert all([b in data.keys() for b in between]) else: between = between[0] if isinstance(within, list): if len(within) > 1: multiple_within = True contrast = 'multiple_within' assert all([w in data.keys() for w in within]) else: within = within[0] if all([multiple_within, multiple_between]): raise ValueError("Multiple between and within factors are", "currently not supported. Please select only one.") # Check the other cases if isinstance(between, str) and within is None: contrast = 'simple_between' assert between in data.keys() if isinstance(within, str) and between is None: contrast = 'simple_within' assert within in data.keys() if isinstance(between, str) and isinstance(within, str): contrast = 'within_between' assert all([between in data.keys(), within in data.keys()]) # Initialize empty variables stats = pd.DataFrame([]) ddic = {} if contrast in ['simple_within', 'simple_between']: # OPTION A: SIMPLE MAIN EFFECTS, WITHIN OR BETWEEN paired = True if contrast == 'simple_within' else False col = within if contrast == 'simple_within' else between # Remove NAN in repeated measurements if contrast == 'simple_within' and data[dv].isnull().values.any(): data = _remove_rm_na(dv=dv, within=within, subject=subject, data=data) # Extract effects labels = data[col].unique().tolist() for l in labels: ddic[l] = data.loc[data[col] == l, dv].values # Number and labels of possible comparisons if len(labels) >= 2: combs = list(combinations(labels, 2)) else: raise ValueError('Columns must have at least two unique values.') # Initialize vectors for comb in combs: col1, col2 = comb x = ddic.get(col1) y = ddic.get(col2) df_ttest = ttest(x, y, paired=paired, tail=tail) ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) stats = _append_stats_dataframe(stats, x, y, col1, col2, alpha, paired, df_ttest, ef, effsize) stats['Contrast'] = col # Multiple comparisons padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': _, stats['p-corr'] = multicomp(stats['p-unc'].values, alpha=alpha, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None else: # B1: BETWEEN1 + BETWEEN2 + BETWEEN1 * BETWEEN2 # B2: WITHIN1 + WITHIN2 + WITHIN1 * WITHIN2 # B3: WITHIN + BETWEEN + WITHIN * BETWEEN if contrast == 'multiple_between': # B1 factors = between fbt = factors fwt = [None, None] # eft = ['between', 'between'] paired = False elif contrast == 'multiple_within': # B2 factors = within fbt = [None, None] fwt = factors # eft = ['within', 'within'] paired = True else: # B3 factors = [within, between] fbt = [None, between] fwt = [within, None] # eft = ['within', 'between'] paired = False for i, f in enumerate(factors): stats = stats.append(pairwise_ttests(dv=dv, between=fbt[i], within=fwt[i], subject=subject, data=data, alpha=alpha, tail=tail, padjust=padjust, effsize=effsize, return_desc=return_desc), ignore_index=True, sort=False) # Then compute the interaction between the factors labels_fac1 = data[factors[0]].unique().tolist() labels_fac2 = data[factors[1]].unique().tolist() comb_fac1 = list(combinations(labels_fac1, 2)) comb_fac2 = list(combinations(labels_fac2, 2)) lc_fac1 = len(comb_fac1) lc_fac2 = len(comb_fac2) for lw in labels_fac1: for l in labels_fac2: tmp = data.loc[data[factors[0]] == lw] ddic[lw, l] = tmp.loc[tmp[factors[1]] == l, dv].values # Pairwise comparisons combs = list(product(labels_fac1, comb_fac2)) for comb in combs: fac1, (col1, col2) = comb x = ddic.get((fac1, col1)) y = ddic.get((fac1, col2)) df_ttest = ttest(x, y, paired=paired, tail=tail) ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) stats = _append_stats_dataframe(stats, x, y, col1, col2, alpha, paired, df_ttest, ef, effsize, fac1) # Update the Contrast columns txt_inter = factors[0] + ' * ' + factors[1] idxitr = np.arange(lc_fac1 + lc_fac2, stats.shape[0]).tolist() stats.loc[idxitr, 'Contrast'] = txt_inter # Multi-comparison columns if padjust is not None and padjust.lower() != 'none': _, pcor = multicomp(stats.loc[idxitr, 'p-unc'].values, alpha=alpha, method=padjust) stats.loc[idxitr, 'p-corr'] = pcor stats.loc[idxitr, 'p-adjust'] = padjust # --------------------------------------------------------------------- stats['Paired'] = stats['Paired'].astype(bool) # Reorganize column order col_order = [ 'Contrast', 'Time', 'A', 'B', 'mean(A)', 'std(A)', 'mean(B)', 'std(B)', 'Paired', 'T', 'tail', 'p-unc', 'p-corr', 'p-adjust', 'BF10', 'efsize', 'eftype' ] if return_desc is False: stats.drop(columns=['mean(A)', 'mean(B)', 'std(A)', 'std(B)'], inplace=True) stats = stats.reindex(columns=col_order) stats.dropna(how='all', axis=1, inplace=True) # Rename Time columns if contrast in ['multiple_within', 'multiple_between', 'within_between']: stats['Time'].fillna('-', inplace=True) stats.rename(columns={'Time': factors[0]}, inplace=True) if export_filename is not None: _export_table(stats, export_filename) return stats
def rcorr(self, method='pearson', upper='pval', decimals=3, padjust=None, stars=True, pval_stars={ 0.001: '***', 0.01: '**', 0.05: '*' }): """ Correlation matrix of a dataframe with p-values and/or sample size on the upper triangle (:py:class:`pandas.DataFrame` method). This method is a faster, but less exhaustive, matrix-version of the :py:func:`pingouin.pairwise_corr` function. It is based on the :py:func:`pandas.DataFrame.corr` method. Missing values are automatically removed from each pairwise correlation. Parameters ---------- self : :py:class:`pandas.DataFrame` Input dataframe. method : str Correlation method. Can be either 'pearson' or 'spearman'. upper : str If 'pval', the upper triangle of the output correlation matrix shows the p-values. If 'n', the upper triangle is the sample size used in each pairwise correlation. decimals : int Number of decimals to display in the output correlation matrix. padjust : string or None Method used for adjustment of pvalues. Available methods are :: 'none' : no correction 'bonf' : one-step Bonferroni correction 'sidak' : one-step Sidak correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction stars : boolean If True, only significant p-values are displayed as stars using the pre-defined thresholds of ``pval_stars``. If False, all the raw p-values are displayed. pval_stars : dict Significance thresholds. Default is 3 stars for p-values < 0.001, 2 stars for p-values < 0.01 and 1 star for p-values < 0.05. Returns ------- rcorr : :py:class:`pandas.DataFrame` Correlation matrix, of type str. Examples -------- >>> import numpy as np >>> import pandas as pd >>> import pingouin as pg >>> # Load an example dataset of personality dimensions >>> df = pg.read_dataset('pairwise_corr').iloc[:, 1:] >>> # Add some missing values >>> df.iloc[[2, 5, 20], 2] = np.nan >>> df.iloc[[1, 4, 10], 3] = np.nan >>> df.head().round(2) Neuroticism Extraversion Openness Agreeableness Conscientiousness 0 2.48 4.21 3.94 3.96 3.46 1 2.60 3.19 3.96 NaN 3.23 2 2.81 2.90 NaN 2.75 3.50 3 2.90 3.56 3.52 3.17 2.79 4 3.02 3.33 4.02 NaN 2.85 >>> # Correlation matrix on the four first columns >>> df.iloc[:, 0:4].rcorr() Neuroticism Extraversion Openness Agreeableness Neuroticism - *** ** Extraversion -0.35 - *** Openness -0.01 0.265 - *** Agreeableness -0.134 0.054 0.161 - >>> # Spearman correlation and Holm adjustement for multiple comparisons >>> df.iloc[:, 0:4].rcorr(method='spearman', padjust='holm') Neuroticism Extraversion Openness Agreeableness Neuroticism - *** ** Extraversion -0.325 - *** Openness -0.027 0.24 - *** Agreeableness -0.15 0.06 0.173 - >>> # Compare with the pg.pairwise_corr function >>> pairwise = df.iloc[:, 0:4].pairwise_corr(method='spearman', ... padjust='holm') >>> pairwise[['X', 'Y', 'r', 'p-corr']].round(3) # Do not show all columns X Y r p-corr 0 Neuroticism Extraversion -0.325 0.000 1 Neuroticism Openness -0.027 0.543 2 Neuroticism Agreeableness -0.150 0.002 3 Extraversion Openness 0.240 0.000 4 Extraversion Agreeableness 0.060 0.358 5 Openness Agreeableness 0.173 0.000 >>> # Display the raw p-values with four decimals >>> df.iloc[:, [0, 1, 3]].rcorr(stars=False, decimals=4) Neuroticism Extraversion Agreeableness Neuroticism - 0.0000 0.0028 Extraversion -0.3501 - 0.2305 Agreeableness -0.134 0.0539 - >>> # With the sample size on the upper triangle instead of the p-values >>> df.iloc[:, [0, 1, 2]].rcorr(upper='n') Neuroticism Extraversion Openness Neuroticism - 500 497 Extraversion -0.35 - 497 Openness -0.01 0.265 - """ from numpy import triu_indices_from as tif from numpy import format_float_positional as ffp from scipy.stats import pearsonr, spearmanr # Safety check assert isinstance(pval_stars, dict), 'pval_stars must be a dictionnary.' assert isinstance(decimals, int), 'decimals must be an int.' assert method in ['pearson', 'spearman'], 'Method is not recognized.' assert upper in ['pval', 'n'], 'upper must be either `pval` or `n`.' mat = self.corr(method=method).round(decimals) if upper == 'n': mat_upper = self.corr(method=lambda x, y: len(x)).astype(int) else: if method == 'pearson': mat_upper = self.corr(method=lambda x, y: pearsonr(x, y)[1]) else: # Method = 'spearman' mat_upper = self.corr(method=lambda x, y: spearmanr(x, y)[1]) if padjust is not None: pvals = mat_upper.values[tif(mat, k=1)] mat_upper.values[tif(mat, k=1)] = multicomp(pvals, alpha=0.05, method=padjust)[1] # Convert r to text mat = mat.astype(str) np.fill_diagonal(mat.values, '-') # Inplace modification of the diagonal if upper == 'pval': def replace_pval(x): for key, value in pval_stars.items(): if x < key: return value return '' if stars: # Replace p-values by stars mat_upper = mat_upper.applymap(replace_pval) else: mat_upper = mat_upper.applymap( lambda x: ffp(x, precision=decimals)) # Replace upper triangle by p-values or n mat.values[tif(mat, k=1)] = mat_upper.values[tif(mat, k=1)] return mat
def pairwise_corr(data, columns=None, covar=None, tail='two-sided', method='pearson', padjust='none', export_filename=None): '''Pairwise (partial) correlations between columns of a pandas dataframe. Parameters ---------- data : pandas DataFrame DataFrame. Note that this function can also directly be used as a Pandas method, in which case this argument is no longer needed. columns : list or str Column names in data :: '["a", "b", "c"]' : combination between columns a, b, and c '["a"]' : product between a and all the other numeric columns '[["a"], ["b", "c"]]' : product between ["a"] and ["b", "c"] '[["a", "d"], ["b", "c"]]' : product between ["a", "d"] and ["b", "c"] '[["a", "d"], None]' : product between ["a", "d"] and all other columns Note that if column is not specified, then the function will return the pairwise correlation between the combination of all the numeric columns in data. See the examples section for more details on this. covar : None, string or list Covariate(s) for partial correlation. Must be one or more columns in data. Use a list if there are more than one covariate. If ``covar`` is not None, a partial correlation will be computed using :py:func:`pingouin.partial_corr` function. tail : string Indicates whether to return the 'two-sided' or 'one-sided' p-values method : string Specify which method to use for the computation of the correlation coefficient. Available methods are :: 'pearson' : Pearson product-moment correlation 'spearman' : Spearman rank-order correlation 'kendall' : Kendall’s tau (ordinal data) 'percbend' : percentage bend correlation (robust) 'shepherd' : Shepherd's pi correlation (robust Spearman) padjust : string Method used for testing and adjustment of pvalues. Available methods are :: 'none' : no correction 'bonferroni' : one-step Bonferroni correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction export_filename : string Filename (without extension) for the output file. If None, do not export the table. By default, the file will be created in the current python console directory. To change that, specify the filename with full path. Returns ------- stats : DataFrame Stats summary :: 'X' : Name(s) of first columns 'Y' : Name(s) of second columns 'method' : method used to compute the correlation 'covar' : List of specified covariate(s) (only for partial correlation) 'tail' : indicates whether the p-values are one-sided or two-sided 'n' : Sample size (after NaN removal) 'r' : Correlation coefficients 'CI95' : 95% parametric confidence intervals 'r2' : R-squared values 'adj_r2' : Adjusted R-squared values 'z' : Standardized correlation coefficients 'p-unc' : uncorrected one or two tailed p-values 'p-corr' : corrected one or two tailed p-values 'p-adjust' : Correction method Notes ----- Please refer to the :py:func:`pingouin.corr()` function for a description of the different methods. NaN are automatically removed from the data. This function is more flexible and gives a much more detailed output than the :py:func:`pandas.DataFrame.corr()` method (i.e. p-values, confidence interval, Bayes Factor..). This comes however at an increased computational cost. While this should not be discernible for dataframe with less than 10,000 rows and/or less than 20 columns, this function can be slow for very large dataset. For speed purpose, the Bayes Factor is only computed when the sample size is less than 1000 (and method='pearson'). This function also works with two-dimensional multi-index columns. In this case, columns must be list(s) of tuple(s). See the Jupyter notebook for more details: https://github.com/raphaelvallat/pingouin/blob/master/notebooks/04_Correlations.ipynb If ``covar`` is specified, this function will compute the pairwise partial correlation between the variables. If you are only interested in computing the partial correlation matrix (i.e. the raw pairwise partial correlation coefficient matrix, without the p-values, sample sizes, etc), a better alternative is to use the :py:func:`pingouin.pcorr` function (see example 7). Examples -------- 1. One-tailed spearman correlation corrected for multiple comparisons >>> from pingouin import pairwise_corr, read_dataset >>> data = read_dataset('pairwise_corr').iloc[:, 1:] >>> pairwise_corr(data, method='spearman', tail='two-sided', ... padjust='bonf') # doctest: +SKIP 2. Robust two-sided correlation with uncorrected p-values >>> pcor = pairwise_corr(data, columns=['Openness', 'Extraversion', ... 'Neuroticism'], method='percbend') 3. One-versus-all pairwise correlations >>> pairwise_corr(data, columns=['Neuroticism']) # doctest: +SKIP 4. Pairwise correlations between two lists of columns (cartesian product) >>> columns = [['Neuroticism', 'Extraversion'], ['Openness']] >>> pairwise_corr(data, columns) # doctest: +SKIP 5. As a Pandas method >>> pcor = data.pairwise_corr(covar='Neuroticism', method='spearman') 6. Pairwise partial correlation >>> pcor = pairwise_corr(data, covar='Neuroticism') # One covariate >>> pcor = pairwise_corr(data, covar=['Neuroticism', 'Openness']) # Two 7. Pairwise partial correlation matrix (only the r-values) >>> data[['Neuroticism', 'Openness', 'Extraversion']].pcorr() Neuroticism Openness Extraversion Neuroticism 1.000000 0.092097 -0.360421 Openness 0.092097 1.000000 0.281312 Extraversion -0.360421 0.281312 1.000000 ''' from pingouin.correlation import corr, partial_corr if tail not in ['one-sided', 'two-sided']: raise ValueError('Tail not recognized') # Keep only numeric columns data = data._get_numeric_data() # Remove columns with constant value and/or NaN data = data.loc[:, data.nunique(dropna=True) >= 2] # Extract columns names keys = data.columns.tolist() # First ensure that columns is a list if isinstance(columns, (str, tuple)): columns = [columns] def traverse(o, tree_types=(list, tuple)): """Helper function to flatten nested lists. From https://stackoverflow.com/a/6340578 """ if isinstance(o, tree_types): for value in o: for subvalue in traverse(value, tree_types): yield subvalue else: yield o # Check if columns index has multiple levels if isinstance(data.columns, pd.core.index.MultiIndex): multi_index = True if columns is not None: # Simple List with one element: [('L0', 'L1')] # Simple list with >= 2 elements: [('L0', 'L1'), ('L0', 'L2')] # Nested lists: [[('L0', 'L1')], ...] or [..., [('L0', 'L1')]] col_flatten = list(traverse(columns, tree_types=list)) assert all(isinstance(c, (tuple, type(None))) for c in col_flatten) else: multi_index = False # Then define combinations / products between columns if columns is None: # Case A: column is not defined --> corr between all numeric columns combs = list(combinations(keys, 2)) else: # Case B: column is specified if isinstance(columns[0], list): group1 = [e for e in columns[0] if e in keys] # Assert that column is two-dimensional if len(columns) == 1: columns.append(None) if isinstance(columns[1], list) and len(columns[1]): # B1: [['a', 'b'], ['c', 'd']] group2 = [e for e in columns[1] if e in keys] else: # B2: [['a', 'b']], [['a', 'b'], None] or [['a', 'b'], 'all'] group2 = [e for e in keys if e not in group1] combs = list(product(group1, group2)) else: # Column is a simple list if len(columns) == 1: # Case B3: one-versus-all, e.g. ['a'] or 'a' # Check that this column exist if columns[0] not in keys: msg = ('"%s" is not in data or is not numeric.' % columns[0]) raise ValueError(msg) others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # Combinations between all specified columns ['a', 'b', 'c'] # Make sure that we keep numeric columns columns = [c for c in columns if c in keys] if len(columns) == 1: # If only one-column is left, equivalent to ['a'] others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # combinations between ['a', 'b', 'c'] combs = list(combinations(columns, 2)) combs = np.array(combs) if len(combs) == 0: raise ValueError("No column combination found. Please make sure that " "the specified columns exist in the dataframe, are " "numeric, and contains at least two unique values.") # Initialize empty dataframe if multi_index: X = list(zip(combs[:, 0, 0], combs[:, 0, 1])) Y = list(zip(combs[:, 1, 0], combs[:, 1, 1])) else: X = combs[:, 0] Y = combs[:, 1] stats = pd.DataFrame({ 'X': X, 'Y': Y, 'method': method, 'tail': tail }, index=range(len(combs)), columns=[ 'X', 'Y', 'method', 'tail', 'n', 'outliers', 'r', 'CI95%', 'r2', 'adj_r2', 'p-val', 'BF10', 'power' ]) # Now we check if covariates are present if covar is not None: assert isinstance(covar, (str, list)), 'covar must be list or string.' if isinstance(covar, str): covar = [covar] # Check that columns exist and are numeric assert all([c in keys for c in covar]), 'covar not in data or not num.' # And we make sure that X or Y does not contain covar stats = stats[~stats[['X', 'Y']].isin(covar).any(1)] stats = stats.reset_index(drop=True) if stats.shape[0] == 0: raise ValueError("No column combination found. Please make sure " "that the specified columns and covar exist in " "the dataframe, are numeric, and contains at " "least two unique values.") # Compute pairwise correlations and fill dataframe dvs = ['n', 'r', 'CI95%', 'r2', 'adj_r2', 'p-val', 'power'] dvs_out = dvs + ['outliers'] dvs_bf10 = dvs + ['BF10'] for i in range(stats.shape[0]): col1, col2 = stats.loc[i, 'X'], stats.loc[i, 'Y'] if covar is None: cor_st = corr(data[col1].values, data[col2].values, tail=tail, method=method) else: cor_st = partial_corr(data=data, x=col1, y=col2, covar=covar, tail=tail, method=method) cor_st_keys = cor_st.columns.tolist() if 'BF10' in cor_st_keys: stats.loc[i, dvs_bf10] = cor_st[dvs_bf10].values elif 'outliers' in cor_st_keys: stats.loc[i, dvs_out] = cor_st[dvs_out].values else: stats.loc[i, dvs] = cor_st[dvs].values # Force conversion to numeric stats = stats.astype({ 'r': float, 'r2': float, 'adj_r2': float, 'n': int, 'p-val': float, 'outliers': float, 'power': float }) # Multiple comparisons stats = stats.rename(columns={'p-val': 'p-unc'}) padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': reject, stats['p-corr'] = multicomp(stats['p-unc'].values, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None # Standardize correlation coefficients (Fisher z-transformation) stats['z'] = np.round(np.arctanh(stats['r'].values), 3) col_order = [ 'X', 'Y', 'method', 'tail', 'n', 'outliers', 'r', 'CI95%', 'r2', 'adj_r2', 'z', 'p-unc', 'p-corr', 'p-adjust', 'BF10', 'power' ] # Reorder columns and remove empty ones stats = stats.reindex(columns=col_order) stats = stats.dropna(how='all', axis=1) # Add covariates names if present if covar is not None: stats.insert(loc=3, column='covar', value=str(covar)) if export_filename is not None: _export_table(stats, export_filename) return stats
def pairwise_ttests(dv=None, between=None, within=None, subject=None, data=None, parametric=True, alpha=.05, tail='two-sided', padjust='none', effsize='hedges', return_desc=False, export_filename=None): '''Pairwise T-tests. Parameters ---------- dv : string Name of column containing the dependant variable. between : string or list with 2 elements Name of column(s) containing the between factor(s). within : string or list with 2 elements Name of column(s) containing the within factor(s). subject : string Name of column containing the subject identifier. Compulsory for contrast including a within-subject factor. data : pandas DataFrame DataFrame. Note that this function can also directly be used as a Pandas method, in which case this argument is no longer needed. parametric : boolean If True (default), use the parametric :py:func:`ttest` function. If False, use :py:func:`pingouin.wilcoxon` or :py:func:`pingouin.mwu` for paired or unpaired samples, respectively. alpha : float Significance level tail : string Specify whether the alternative hypothesis is `'two-sided'` or `'one-sided'`. Can also be `'greater'` or `'less'` to specify the direction of the test. `'greater'` tests the alternative that ``x`` has a larger mean than ``y``. If tail is `'one-sided'`, Pingouin will automatically infer the one-sided alternative hypothesis of the test based on the test statistic. padjust : string Method used for testing and adjustment of pvalues. Available methods are :: 'none' : no correction 'bonferroni' : one-step Bonferroni correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction effsize : string or None Effect size type. Available methods are :: 'none' : no effect size 'cohen' : Unbiased Cohen d 'hedges' : Hedges g 'glass': Glass delta 'eta-square' : Eta-square 'odds-ratio' : Odds ratio 'AUC' : Area Under the Curve return_desc : boolean If True, append group means and std to the output dataframe export_filename : string Filename (without extension) for the output file. If None, do not export the table. By default, the file will be created in the current python console directory. To change that, specify the filename with full path. Returns ------- stats : DataFrame Stats summary :: 'A' : Name of first measurement 'B' : Name of second measurement 'Paired' : indicates whether the two measurements are paired or not 'Parametric' : indicates if (non)-parametric tests were used 'Tail' : indicate whether the p-values are one-sided or two-sided 'T' : T-values (only if parametric=True) 'U' : Mann-Whitney U value (only if parametric=False and unpaired data) 'W' : Wilcoxon W value (only if parametric=False and paired data) 'dof' : degrees of freedom (only if parametric=True) 'p-unc' : Uncorrected p-values 'p-corr' : Corrected p-values 'p-adjust' : p-values correction method 'BF10' : Bayes Factor 'hedges' : Hedges effect size 'CLES' : Common language effect size See also -------- ttest, mwu, wilcoxon, compute_effsize, multicomp Notes ----- Data are expected to be in long-format. If your data is in wide-format, you can use the :py:func:`pandas.melt` function to convert from wide to long format. If ``between`` or ``within`` is a list (e.g. ['col1', 'col2']), the function returns 1) the pairwise T-tests between each values of the first column, 2) the pairwise T-tests between each values of the second column and 3) the interaction between col1 and col2. The interaction is dependent of the order of the list, so ['col1', 'col2'] will not yield the same results as ['col2', 'col1']. In other words, if ``between`` is a list with two elements, the output model is between1 + between2 + between1 * between2. Similarly, if `within`` is a list with two elements, the output model is within1 + within2 + within1 * within2. If both ``between`` and ``within`` are specified, the function return within + between + within * between. Missing values in repeated measurements are automatically removed using the :py:func:`pingouin.remove_rm_na` function. However, you should be very careful since it can result in undesired values removal (especially for the interaction effect). We strongly recommend that you preprocess your data and remove the missing values before using this function. This function has been tested against the `pairwise.t.test` R function. Examples -------- 1. One between-factor >>> from pingouin import pairwise_ttests, read_dataset >>> df = read_dataset('mixed_anova.csv') >>> post_hocs = pairwise_ttests(dv='Scores', between='Group', data=df) 2. One within-factor >>> post_hocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', data=df) >>> print(post_hocs) # doctest: +SKIP 3. Non-parametric pairwise paired test (wilcoxon) >>> pairwise_ttests(dv='Scores', within='Time', subject='Subject', ... data=df, parametric=False) # doctest: +SKIP 4. Within + Between + Within * Between with corrected p-values >>> posthocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', between='Group', ... padjust='bonf', data=df) 5. Between1 + Between2 + Between1 * Between2 >>> posthocs = pairwise_ttests(dv='Scores', between=['Group', 'Time'], ... data=df) ''' from .parametric import ttest from .nonparametric import wilcoxon, mwu # Safety checks _check_dataframe(dv=dv, between=between, within=within, subject=subject, effects='all', data=data) if tail not in ['one-sided', 'two-sided', 'greater', 'less']: raise ValueError('Tail not recognized') if not isinstance(alpha, float): raise ValueError('Alpha must be float') # Check if we have multiple between or within factors multiple_between = False multiple_within = False contrast = None if isinstance(between, list): if len(between) > 1: multiple_between = True contrast = 'multiple_between' assert all([b in data.keys() for b in between]) else: between = between[0] if isinstance(within, list): if len(within) > 1: multiple_within = True contrast = 'multiple_within' assert all([w in data.keys() for w in within]) else: within = within[0] if all([multiple_within, multiple_between]): raise ValueError("Multiple between and within factors are", "currently not supported. Please select only one.") # Check the other cases if isinstance(between, str) and within is None: contrast = 'simple_between' assert between in data.keys() if isinstance(within, str) and between is None: contrast = 'simple_within' assert within in data.keys() if isinstance(between, str) and isinstance(within, str): contrast = 'within_between' assert all([between in data.keys(), within in data.keys()]) # Initialize empty variables stats = pd.DataFrame([]) ddic = {} if contrast in ['simple_within', 'simple_between']: # OPTION A: SIMPLE MAIN EFFECTS, WITHIN OR BETWEEN paired = True if contrast == 'simple_within' else False col = within if contrast == 'simple_within' else between # Remove NAN in repeated measurements if contrast == 'simple_within' and data[dv].isnull().values.any(): data = remove_rm_na(dv=dv, within=within, subject=subject, data=data) # Extract effects labels = data[col].unique().tolist() for l in labels: ddic[l] = data.loc[data[col] == l, dv].values # Number and labels of possible comparisons if len(labels) >= 2: combs = list(combinations(labels, 2)) else: raise ValueError('Columns must have at least two unique values.') # Initialize vectors for comb in combs: col1, col2 = comb x = ddic.get(col1) y = ddic.get(col2) if parametric: df_ttest = ttest(x, y, paired=paired, tail=tail) # Compute exact CLES df_ttest['CLES'] = compute_effsize(x, y, paired=paired, eftype='CLES') else: if paired: df_ttest = wilcoxon(x, y, tail=tail) else: df_ttest = mwu(x, y, tail=tail) # Compute Hedges / Cohen ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) stats = _append_stats_dataframe(stats, x, y, col1, col2, alpha, paired, tail, df_ttest, ef, effsize) stats['Contrast'] = col # Multiple comparisons padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': _, stats['p-corr'] = multicomp(stats['p-unc'].values, alpha=alpha, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None else: # B1: BETWEEN1 + BETWEEN2 + BETWEEN1 * BETWEEN2 # B2: WITHIN1 + WITHIN2 + WITHIN1 * WITHIN2 # B3: WITHIN + BETWEEN + WITHIN * BETWEEN if contrast == 'multiple_between': # B1 factors = between fbt = factors fwt = [None, None] # eft = ['between', 'between'] paired = False elif contrast == 'multiple_within': # B2 factors = within fbt = [None, None] fwt = factors # eft = ['within', 'within'] paired = True else: # B3 factors = [within, between] fbt = [None, between] fwt = [within, None] # eft = ['within', 'between'] paired = False for i, f in enumerate(factors): stats = stats.append(pairwise_ttests(dv=dv, between=fbt[i], within=fwt[i], subject=subject, data=data, parametric=parametric, alpha=alpha, tail=tail, padjust=padjust, effsize=effsize, return_desc=return_desc), ignore_index=True, sort=False) # Rename effect size to generic name stats.rename(columns={effsize: 'efsize'}, inplace=True) # Then compute the interaction between the factors labels_fac1 = data[factors[0]].unique().tolist() labels_fac2 = data[factors[1]].unique().tolist() comb_fac1 = list(combinations(labels_fac1, 2)) comb_fac2 = list(combinations(labels_fac2, 2)) lc_fac1 = len(comb_fac1) lc_fac2 = len(comb_fac2) for lw in labels_fac1: for l in labels_fac2: tmp = data.loc[data[factors[0]] == lw] ddic[lw, l] = tmp.loc[tmp[factors[1]] == l, dv].values # Pairwise comparisons combs = list(product(labels_fac1, comb_fac2)) for comb in combs: fac1, (col1, col2) = comb x = ddic.get((fac1, col1)) y = ddic.get((fac1, col2)) if parametric: df_ttest = ttest(x, y, paired=paired, tail=tail) # Compute exact CLES df_ttest['CLES'] = compute_effsize(x, y, paired=paired, eftype='CLES') else: if paired: df_ttest = wilcoxon(x, y, tail=tail) else: df_ttest = mwu(x, y, tail=tail) ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) stats = _append_stats_dataframe(stats, x, y, col1, col2, alpha, paired, tail, df_ttest, ef, effsize, fac1) # Update the Contrast columns txt_inter = factors[0] + ' * ' + factors[1] idxitr = np.arange(lc_fac1 + lc_fac2, stats.shape[0]).tolist() stats.loc[idxitr, 'Contrast'] = txt_inter # Multi-comparison columns if padjust is not None and padjust.lower() != 'none': _, pcor = multicomp(stats.loc[idxitr, 'p-unc'].values, alpha=alpha, method=padjust) stats.loc[idxitr, 'p-corr'] = pcor stats.loc[idxitr, 'p-adjust'] = padjust # --------------------------------------------------------------------- stats['Paired'] = stats['Paired'].astype(bool) stats['Parametric'] = parametric # Round effect size and CLES stats[['efsize', 'CLES']] = stats[['efsize', 'CLES']].round(3) # Reorganize column order col_order = [ 'Contrast', 'Time', 'A', 'B', 'mean(A)', 'std(A)', 'mean(B)', 'std(B)', 'Paired', 'Parametric', 'T', 'U', 'W', 'dof', 'tail', 'p-unc', 'p-corr', 'p-adjust', 'BF10', 'CLES', 'efsize' ] if return_desc is False: stats.drop(columns=['mean(A)', 'mean(B)', 'std(A)', 'std(B)'], inplace=True) stats = stats.reindex(columns=col_order) stats.dropna(how='all', axis=1, inplace=True) # Rename effect size column stats.rename(columns={'efsize': effsize}, inplace=True) # Rename Time columns if contrast in ['multiple_within', 'multiple_between', 'within_between']: stats['Time'].fillna('-', inplace=True) stats.rename(columns={'Time': factors[0]}, inplace=True) if export_filename is not None: _export_table(stats, export_filename) return stats
def pairwise_ttests(data=None, dv=None, between=None, within=None, subject=None, parametric=True, alpha=.05, tail='two-sided', padjust='none', effsize='hedges', nan_policy='listwise', return_desc=False, interaction=True, export_filename=None): '''Pairwise T-tests. Parameters ---------- data : pandas DataFrame DataFrame. Note that this function can also directly be used as a Pandas method, in which case this argument is no longer needed. dv : string Name of column containing the dependant variable. between : string or list with 2 elements Name of column(s) containing the between factor(s). within : string or list with 2 elements Name of column(s) containing the within factor(s). subject : string Name of column containing the subject identifier. Compulsory for contrast including a within-subject factor. parametric : boolean If True (default), use the parametric :py:func:`ttest` function. If False, use :py:func:`pingouin.wilcoxon` or :py:func:`pingouin.mwu` for paired or unpaired samples, respectively. alpha : float Significance level tail : string Specify whether the alternative hypothesis is `'two-sided'` or `'one-sided'`. Can also be `'greater'` or `'less'` to specify the direction of the test. `'greater'` tests the alternative that ``x`` has a larger mean than ``y``. If tail is `'one-sided'`, Pingouin will automatically infer the one-sided alternative hypothesis of the test based on the test statistic. padjust : string Method used for testing and adjustment of pvalues. Available methods are :: 'none' : no correction 'bonf' : one-step Bonferroni correction 'sidak' : one-step Sidak correction 'holm' : step-down method using Bonferroni adjustments 'fdr_bh' : Benjamini/Hochberg FDR correction 'fdr_by' : Benjamini/Yekutieli FDR correction effsize : string or None Effect size type. Available methods are :: 'none' : no effect size 'cohen' : Unbiased Cohen d 'hedges' : Hedges g 'glass': Glass delta 'r' : Pearson correlation coefficient 'eta-square' : Eta-square 'odds-ratio' : Odds ratio 'AUC' : Area Under the Curve 'CLES' : Common Language Effect Size nan_policy : string Can be `'listwise'` for listwise deletion of missing values in repeated measures design (= complete-case analysis) or `'pairwise'` for the more liberal pairwise deletion (= available-case analysis). .. versionadded:: 0.2.9 return_desc : boolean If True, append group means and std to the output dataframe interaction : boolean If there are multiple factors and ``interaction`` is True (default), Pingouin will also calculate T-tests for the interaction term (see Notes). .. versionadded:: 0.2.9 export_filename : string Filename (without extension) for the output file. If None, do not export the table. By default, the file will be created in the current python console directory. To change that, specify the filename with full path. Returns ------- stats : DataFrame Stats summary :: 'A' : Name of first measurement 'B' : Name of second measurement 'Paired' : indicates whether the two measurements are paired or not 'Parametric' : indicates if (non)-parametric tests were used 'Tail' : indicate whether the p-values are one-sided or two-sided 'T' : T statistic (only if parametric=True) 'U-val' : Mann-Whitney U stat (if parametric=False and unpaired data) 'W-val' : Wilcoxon W stat (if parametric=False and paired data) 'dof' : degrees of freedom (only if parametric=True) 'p-unc' : Uncorrected p-values 'p-corr' : Corrected p-values 'p-adjust' : p-values correction method 'BF10' : Bayes Factor 'hedges' : effect size (or any effect size defined in ``effsize``) See also -------- ttest, mwu, wilcoxon, compute_effsize, multicomp Notes ----- Data are expected to be in long-format. If your data is in wide-format, you can use the :py:func:`pandas.melt` function to convert from wide to long format. If ``between`` or ``within`` is a list (e.g. ['col1', 'col2']), the function returns 1) the pairwise T-tests between each values of the first column, 2) the pairwise T-tests between each values of the second column and 3) the interaction between col1 and col2. The interaction is dependent of the order of the list, so ['col1', 'col2'] will not yield the same results as ['col2', 'col1'], and will only be calculated if ``interaction=True``. In other words, if ``between`` is a list with two elements, the output model is between1 + between2 + between1 * between2. Similarly, if `within`` is a list with two elements, the output model is within1 + within2 + within1 * within2. If both ``between`` and ``within`` are specified, the function return within + between + within * between. Missing values in repeated measurements are automatically removed using a listwise (default) or pairwise deletion strategy. However, you should be very careful since it can result in undesired values removal (especially for the interaction effect). We strongly recommend that you preprocess your data and remove the missing values before using this function. This function has been tested against the `pairwise.t.test` R function. Examples -------- 1. One between-factor >>> from pingouin import pairwise_ttests, read_dataset >>> df = read_dataset('mixed_anova.csv') >>> post_hocs = pairwise_ttests(dv='Scores', between='Group', data=df) 2. One within-factor >>> post_hocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', data=df) >>> print(post_hocs) # doctest: +SKIP 3. Non-parametric pairwise paired test (wilcoxon) >>> pairwise_ttests(dv='Scores', within='Time', subject='Subject', ... data=df, parametric=False) # doctest: +SKIP 4. Within + Between + Within * Between with corrected p-values >>> posthocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', between='Group', ... padjust='bonf', data=df) 5. Between1 + Between2 + Between1 * Between2 >>> posthocs = pairwise_ttests(dv='Scores', between=['Group', 'Time'], ... data=df) 6. Between1 + Between2, no interaction >>> posthocs = df.pairwise_ttests(dv='Scores', between=['Group', 'Time'], ... interaction=False) ''' from .parametric import ttest from .nonparametric import wilcoxon, mwu # Safety checks _check_dataframe(dv=dv, between=between, within=within, subject=subject, effects='all', data=data) assert tail in ['one-sided', 'two-sided', 'greater', 'less'] assert isinstance(alpha, float), 'alpha must be float.' assert nan_policy in ['listwise', 'pairwise'] # Check if we have multiple between or within factors multiple_between = False multiple_within = False contrast = None if isinstance(between, list): if len(between) > 1: multiple_between = True contrast = 'multiple_between' assert all([b in data.keys() for b in between]) else: between = between[0] if isinstance(within, list): if len(within) > 1: multiple_within = True contrast = 'multiple_within' assert all([w in data.keys() for w in within]) else: within = within[0] if all([multiple_within, multiple_between]): raise ValueError("Multiple between and within factors are", "currently not supported. Please select only one.") # Check the other cases if isinstance(between, str) and within is None: contrast = 'simple_between' assert between in data.keys() if isinstance(within, str) and between is None: contrast = 'simple_within' assert within in data.keys() if isinstance(between, str) and isinstance(within, str): contrast = 'within_between' assert all([between in data.keys(), within in data.keys()]) # Reorganize column order col_order = [ 'Contrast', 'Time', 'A', 'B', 'mean(A)', 'std(A)', 'mean(B)', 'std(B)', 'Paired', 'Parametric', 'T', 'U-val', 'W-val', 'dof', 'Tail', 'p-unc', 'p-corr', 'p-adjust', 'BF10', effsize ] if contrast in ['simple_within', 'simple_between']: # OPTION A: SIMPLE MAIN EFFECTS, WITHIN OR BETWEEN paired = True if contrast == 'simple_within' else False col = within if contrast == 'simple_within' else between # Remove NAN in repeated measurements if contrast == 'simple_within' and data[dv].isnull().values.any(): # Only if nan_policy == 'listwise'. For pairwise deletion, # missing values will be removed directly in the lower-level # functions (e.g. pg.ttest) if nan_policy == 'listwise': data = remove_rm_na(dv=dv, within=within, subject=subject, data=data) else: # The `remove_rm_na` also aggregate other repeated measures # factor using the mean. Here, we ensure this behavior too. data = data.groupby([subject, within])[dv].mean().reset_index() # Now we check that subjects are present in all conditions # For example, if we have four subjects and 3 conditions, # and if subject 2 have missing data at the third condition, # we still need a row with missing values for this subject. if data.groupby(within)[subject].count().nunique() != 1: raise ValueError("Repeated measures dataframe is not balanced." " `Subjects` must have the same number of " "elements in all conditions, " "even when missing values are present.") # Extract effects grp_col = data.groupby(col, sort=False)[dv] labels = grp_col.groups.keys() # Number and labels of possible comparisons if len(labels) >= 2: combs = list(combinations(labels, 2)) combs = np.array(combs) A = combs[:, 0] B = combs[:, 1] else: raise ValueError('Columns must have at least two unique values.') # Initialize dataframe stats = pd.DataFrame(dtype=np.float64, index=range(len(combs)), columns=col_order) # Force dtype conversion cols_str = ['Contrast', 'Time', 'A', 'B', 'Tail', 'p-adjust', 'BF10'] cols_bool = ['Parametric', 'Paired'] stats[cols_str] = stats[cols_str].astype(object) stats[cols_bool] = stats[cols_bool].astype(bool) # Fill str columns stats.loc[:, 'A'] = A stats.loc[:, 'B'] = B stats.loc[:, 'Contrast'] = col stats.loc[:, 'Tail'] = tail stats.loc[:, 'Paired'] = paired for i in range(stats.shape[0]): col1, col2 = stats.at[i, 'A'], stats.at[i, 'B'] x = grp_col.get_group(col1).to_numpy(dtype=np.float64) y = grp_col.get_group(col2).to_numpy(dtype=np.float64) if parametric: stat_name = 'T' df_ttest = ttest(x, y, paired=paired, tail=tail) stats.at[i, 'BF10'] = df_ttest.at['T-test', 'BF10'] stats.at[i, 'dof'] = df_ttest.at['T-test', 'dof'] else: if paired: stat_name = 'W-val' df_ttest = wilcoxon(x, y, tail=tail) else: stat_name = 'U-val' df_ttest = mwu(x, y, tail=tail) # Compute Hedges / Cohen ef = np.round( compute_effsize(x=x, y=y, eftype=effsize, paired=paired), 3) if return_desc: stats.at[i, 'mean(A)'] = np.round(np.nanmean(x), 3) stats.at[i, 'mean(B)'] = np.round(np.nanmean(y), 3) stats.at[i, 'std(A)'] = np.round(np.nanstd(x), 3) stats.at[i, 'std(B)'] = np.round(np.nanstd(y), 3) stats.at[i, stat_name] = df_ttest[stat_name].iat[0] stats.at[i, 'p-unc'] = df_ttest['p-val'].iat[0] stats.at[i, effsize] = ef # Multiple comparisons padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': _, stats['p-corr'] = multicomp(stats['p-unc'].values, alpha=alpha, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None else: # B1: BETWEEN1 + BETWEEN2 + BETWEEN1 * BETWEEN2 # B2: WITHIN1 + WITHIN2 + WITHIN1 * WITHIN2 # B3: WITHIN + BETWEEN + WITHIN * BETWEEN if contrast == 'multiple_between': # B1 factors = between fbt = factors fwt = [None, None] # eft = ['between', 'between'] paired = False elif contrast == 'multiple_within': # B2 factors = within fbt = [None, None] fwt = factors # eft = ['within', 'within'] paired = True else: # B3 factors = [within, between] fbt = [None, between] fwt = [within, None] # eft = ['within', 'between'] paired = False stats = pd.DataFrame() for i, f in enumerate(factors): stats = stats.append(pairwise_ttests(dv=dv, between=fbt[i], within=fwt[i], subject=subject, data=data, parametric=parametric, alpha=alpha, tail=tail, padjust=padjust, effsize=effsize, return_desc=return_desc), ignore_index=True, sort=False) # Then compute the interaction between the factors if interaction: nrows = stats.shape[0] grp_fac1 = data.groupby(factors[0], sort=False)[dv] grp_fac2 = data.groupby(factors[1], sort=False)[dv] grp_both = data.groupby(factors, sort=False)[dv] labels_fac1 = grp_fac1.groups.keys() labels_fac2 = grp_fac2.groups.keys() # comb_fac1 = list(combinations(labels_fac1, 2)) comb_fac2 = list(combinations(labels_fac2, 2)) # Pairwise comparisons combs_list = list(product(labels_fac1, comb_fac2)) ncombs = len(combs_list) # np.array(combs_list) does not work because of tuples # we therefore need to flatten the tupple combs = np.zeros(shape=(ncombs, 3), dtype=object) for i in range(ncombs): combs[i] = _flatten_list(combs_list[i], include_tuple=True) # Append empty rows idxiter = np.arange(nrows, nrows + ncombs) stats = stats.append(pd.DataFrame(columns=stats.columns, index=idxiter), ignore_index=True) # Update other columns stats.loc[idxiter, 'Contrast'] = factors[0] + ' * ' + factors[1] stats.loc[idxiter, 'Time'] = combs[:, 0] stats.loc[idxiter, 'Paired'] = paired stats.loc[idxiter, 'Tail'] = tail stats.loc[idxiter, 'A'] = combs[:, 1] stats.loc[idxiter, 'B'] = combs[:, 2] for i, comb in enumerate(combs): ic = nrows + i # Take into account previous rows fac1, col1, col2 = comb x = grp_both.get_group((fac1, col1)).to_numpy(dtype=np.float64) y = grp_both.get_group((fac1, col2)).to_numpy(dtype=np.float64) ef = np.round( compute_effsize(x=x, y=y, eftype=effsize, paired=paired), 3) if parametric: stat_name = 'T' df_ttest = ttest(x, y, paired=paired, tail=tail) stats.at[ic, 'BF10'] = df_ttest.at['T-test', 'BF10'] stats.at[ic, 'dof'] = df_ttest.at['T-test', 'dof'] else: if paired: stat_name = 'W-val' df_ttest = wilcoxon(x, y, tail=tail) else: stat_name = 'U-val' df_ttest = mwu(x, y, tail=tail) # Append to stats if return_desc: stats.at[ic, 'mean(A)'] = np.round(np.nanmean(x), 3) stats.at[ic, 'mean(B)'] = np.round(np.nanmean(y), 3) stats.at[ic, 'std(A)'] = np.round(np.nanstd(x), 3) stats.at[ic, 'std(B)'] = np.round(np.nanstd(y), 3) stats.at[ic, stat_name] = df_ttest[stat_name].iat[0] stats.at[ic, 'p-unc'] = df_ttest['p-val'].iat[0] stats.at[ic, effsize] = ef # Multi-comparison columns if padjust is not None and padjust.lower() != 'none': _, pcor = multicomp(stats.loc[idxiter, 'p-unc'].values, alpha=alpha, method=padjust) stats.loc[idxiter, 'p-corr'] = pcor stats.loc[idxiter, 'p-adjust'] = padjust # --------------------------------------------------------------------- # Append parametric columns stats.loc[:, 'Parametric'] = parametric # Reorder and drop empty columns stats = stats[np.array(col_order)[np.isin(col_order, stats.columns)]] stats = stats.dropna(how='all', axis=1) # Rename Time columns if (contrast in ['multiple_within', 'multiple_between', 'within_between'] and interaction): stats['Time'].fillna('-', inplace=True) stats.rename(columns={'Time': factors[0]}, inplace=True) if export_filename is not None: _export_table(stats, export_filename) return stats
def pairwise_corr(data, columns=None, covar=None, tail='two-sided', method='pearson', padjust='none', nan_policy='pairwise'): """Pairwise (partial) correlations between columns of a pandas dataframe. Parameters ---------- data : :py:class:`pandas.DataFrame` DataFrame. Note that this function can also directly be used as a Pandas method, in which case this argument is no longer needed. columns : list or str Column names in data: * ``["a", "b", "c"]``: combination between columns a, b, and c. * ``["a"]``: product between a and all the other numeric columns. * ``[["a"], ["b", "c"]]``: product between ["a"] and ["b", "c"]. * ``[["a", "d"], ["b", "c"]]``: product between ["a", "d"] and ["b", "c"]. * ``[["a", "d"], None]``: product between ["a", "d"] and all other numeric columns in dataframe. If column is None, the function will return the pairwise correlation between the combination of all the numeric columns in data. See the examples section for more details on this. covar : None, string or list Covariate(s) for partial correlation. Must be one or more columns in data. Use a list if there are more than one covariate. If ``covar`` is not None, a partial correlation will be computed using :py:func:`pingouin.partial_corr` function. tail : string Specify whether to return ``'one-sided'`` or ``'two-sided'`` p-value. Note that the former are simply half the latter. method : string Correlation type: * ``'pearson'``: Pearson :math:`r` product-moment correlation * ``'spearman'``: Spearman :math:`\\rho` rank-order correlation * ``'kendall'``: Kendall's :math:`\\tau` correlation (for ordinal data) * ``'bicor'``: Biweight midcorrelation (robust) * ``'percbend'``: Percentage bend correlation (robust) * ``'shepherd'``: Shepherd's pi correlation (robust) * ``'skipped'``: Skipped correlation (robust) padjust : string Method used for testing and adjustment of pvalues. * ``'none'``: no correction * ``'bonf'``: one-step Bonferroni correction * ``'sidak'``: one-step Sidak correction * ``'holm'``: step-down method using Bonferroni adjustments * ``'fdr_bh'``: Benjamini/Hochberg FDR correction * ``'fdr_by'``: Benjamini/Yekutieli FDR correction nan_policy : string Can be ``'listwise'`` for listwise deletion of missing values (= complete-case analysis) or ``'pairwise'`` (default) for the more liberal pairwise deletion (= available-case analysis). .. versionadded:: 0.2.9 Returns ------- stats : :py:class:`pandas.DataFrame` * ``'X'``: Name(s) of first columns. * ``'Y'``: Name(s) of second columns. * ``'method'``: Correlation type. * ``'covar'``: List of specified covariate(s), only when covariates are passed. * ``'tail'``: Tail of the test. * ``'n'``: Sample size (after removal of missing values). * ``'r'``: Correlation coefficients. * ``'CI95'``: 95% parametric confidence intervals. * ``'r2'``: R-squared values. * ``'adj_r2'``: Adjusted R-squared values. * ``'z'``: Standardized correlation coefficients. * ``'p-unc'``: Uncorrected p-values. * ``'p-corr'``: Corrected p-values. * ``'p-adjust'``: P-values correction method. * ``'BF10'``: Bayes Factor of the alternative hypothesis (only for Pearson correlation) * ``'power'``: achieved power of the test (= 1 - type II error). Notes ----- Please refer to the :py:func:`pingouin.corr()` function for a description of the different methods. NaN are automatically removed from the data using a pairwise deletion. This function is more flexible and gives a much more detailed output than the :py:func:`pandas.DataFrame.corr()` method (i.e. p-values, confidence interval, Bayes Factor...). This comes however at an increased computational cost. While this should not be discernible for dataframe with less than 10,000 rows and/or less than 20 columns, this function can be slow for very large dataset. A faster alternative to get the r-values and p-values in a matrix format is to use the :py:func:`pingouin.rcorr` function, which works directly as a :py:class:`pandas.DataFrame` method (see example below). This function also works with two-dimensional multi-index columns. In this case, columns must be list(s) of tuple(s). Please refer to this `example Jupyter notebook <https://github.com/raphaelvallat/pingouin/blob/master/notebooks/04_Correlations.ipynb>`_ for more details. If ``covar`` is specified, this function will compute the pairwise partial correlation between the variables. If you are only interested in computing the partial correlation matrix (i.e. the raw pairwise partial correlation coefficient matrix, without the p-values, sample sizes, etc), a better alternative is to use the :py:func:`pingouin.pcorr` function (see example 7). Examples -------- 1. One-sided spearman correlation corrected for multiple comparisons >>> from pingouin import pairwise_corr, read_dataset >>> data = read_dataset('pairwise_corr').iloc[:, 1:] >>> pairwise_corr(data, method='spearman', tail='one-sided', ... padjust='bonf') # doctest: +SKIP 2. Robust two-sided biweight midcorrelation with uncorrected p-values >>> pcor = pairwise_corr(data, columns=['Openness', 'Extraversion', ... 'Neuroticism'], method='bicor') 3. One-versus-all pairwise correlations >>> pairwise_corr(data, columns=['Neuroticism']) # doctest: +SKIP 4. Pairwise correlations between two lists of columns (cartesian product) >>> columns = [['Neuroticism', 'Extraversion'], ['Openness']] >>> pairwise_corr(data, columns) # doctest: +SKIP 5. As a Pandas method >>> pcor = data.pairwise_corr(covar='Neuroticism', method='spearman') 6. Pairwise partial correlation >>> pcor = pairwise_corr(data, covar='Neuroticism') # One covariate >>> pcor = pairwise_corr(data, covar=['Neuroticism', 'Openness']) # Two 7. Pairwise partial correlation matrix using :py:func:`pingouin.pcorr` >>> data[['Neuroticism', 'Openness', 'Extraversion']].pcorr() Neuroticism Openness Extraversion Neuroticism 1.000000 0.092097 -0.360421 Openness 0.092097 1.000000 0.281312 Extraversion -0.360421 0.281312 1.000000 8. Correlation matrix with p-values using :py:func:`pingouin.rcorr` >>> data[['Neuroticism', 'Openness', 'Extraversion']].rcorr() Neuroticism Openness Extraversion Neuroticism - *** Openness -0.01 - *** Extraversion -0.35 0.267 - """ from pingouin.correlation import corr, partial_corr # Check arguments assert tail in ['one-sided', 'two-sided'] assert nan_policy in ['listwise', 'pairwise'] # Keep only numeric columns data = data._get_numeric_data() # Remove columns with constant value and/or NaN data = data.loc[:, data.nunique(dropna=True) >= 2] # Extract columns names keys = data.columns.tolist() # First ensure that columns is a list if isinstance(columns, (str, tuple)): columns = [columns] def traverse(o, tree_types=(list, tuple)): """Helper function to flatten nested lists. From https://stackoverflow.com/a/6340578 """ if isinstance(o, tree_types): for value in o: for subvalue in traverse(value, tree_types): yield subvalue else: yield o # Check if columns index has multiple levels pdv = pd.__version__ mindex = pd.MultiIndex if pdv.startswith('1') else pd.core.index.MultiIndex if isinstance(data.columns, mindex): multi_index = True if columns is not None: # Simple List with one element: [('L0', 'L1')] # Simple list with >= 2 elements: [('L0', 'L1'), ('L0', 'L2')] # Nested lists: [[('L0', 'L1')], ...] or [..., [('L0', 'L1')]] col_flatten = list(traverse(columns, tree_types=list)) assert all(isinstance(c, (tuple, type(None))) for c in col_flatten) else: multi_index = False # Then define combinations / products between columns if columns is None: # Case A: column is not defined --> corr between all numeric columns combs = list(combinations(keys, 2)) else: # Case B: column is specified if isinstance(columns[0], list): group1 = [e for e in columns[0] if e in keys] # Assert that column is two-dimensional if len(columns) == 1: columns.append(None) if isinstance(columns[1], list) and len(columns[1]): # B1: [['a', 'b'], ['c', 'd']] group2 = [e for e in columns[1] if e in keys] else: # B2: [['a', 'b']], [['a', 'b'], None] or [['a', 'b'], 'all'] group2 = [e for e in keys if e not in group1] combs = list(product(group1, group2)) else: # Column is a simple list if len(columns) == 1: # Case B3: one-versus-all, e.g. ['a'] or 'a' # Check that this column exist if columns[0] not in keys: msg = ('"%s" is not in data or is not numeric.' % columns[0]) raise ValueError(msg) others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # Combinations between all specified columns ['a', 'b', 'c'] # Make sure that we keep numeric columns columns = [c for c in columns if c in keys] if len(columns) == 1: # If only one-column is left, equivalent to ['a'] others = [e for e in keys if e != columns[0]] combs = list(product(columns, others)) else: # combinations between ['a', 'b', 'c'] combs = list(combinations(columns, 2)) combs = np.array(combs) if len(combs) == 0: raise ValueError("No column combination found. Please make sure that " "the specified columns exist in the dataframe, are " "numeric, and contains at least two unique values.") # Initialize empty dataframe if multi_index: X = list(zip(combs[:, 0, 0], combs[:, 0, 1])) Y = list(zip(combs[:, 1, 0], combs[:, 1, 1])) else: X = combs[:, 0] Y = combs[:, 1] stats = pd.DataFrame({'X': X, 'Y': Y, 'method': method, 'tail': tail}, index=range(len(combs)), columns=['X', 'Y', 'method', 'tail', 'n', 'outliers', 'r', 'CI95%', 'r2', 'adj_r2', 'p-val', 'BF10', 'power']) # Now we check if covariates are present if covar is not None: assert isinstance(covar, (str, list)), 'covar must be list or string.' if isinstance(covar, str): covar = [covar] # Check that columns exist and are numeric assert all([c in keys for c in covar]), 'covar not in data or not num.' # And we make sure that X or Y does not contain covar stats = stats[~stats[['X', 'Y']].isin(covar).any(1)] stats = stats.reset_index(drop=True) if stats.shape[0] == 0: raise ValueError("No column combination found. Please make sure " "that the specified columns and covar exist in " "the dataframe, are numeric, and contains at " "least two unique values.") # Listwise deletion of missing values if nan_policy == 'listwise': all_cols = np.unique(stats[['X', 'Y']].to_numpy()).tolist() if covar is not None: all_cols.extend(covar) data = data[all_cols].dropna() # Compute pairwise correlations and fill dataframe dvs = ['n', 'r', 'CI95%', 'r2', 'adj_r2', 'p-val', 'power'] dvs_out = dvs + ['outliers'] dvs_bf10 = dvs + ['BF10'] for i in range(stats.shape[0]): col1, col2 = stats.at[i, 'X'], stats.at[i, 'Y'] if covar is None: cor_st = corr(data[col1].to_numpy(), data[col2].to_numpy(), tail=tail, method=method) else: cor_st = partial_corr(data=data, x=col1, y=col2, covar=covar, tail=tail, method=method) cor_st_keys = cor_st.columns.tolist() if 'BF10' in cor_st_keys: stats.loc[i, dvs_bf10] = cor_st[dvs_bf10].to_numpy() elif 'outliers' in cor_st_keys: stats.loc[i, dvs_out] = cor_st[dvs_out].to_numpy() else: stats.loc[i, dvs] = cor_st[dvs].to_numpy() # Force conversion to numeric stats = stats.astype({'r': float, 'r2': float, 'adj_r2': float, 'n': int, 'p-val': float, 'outliers': float, 'power': float}) # Multiple comparisons stats = stats.rename(columns={'p-val': 'p-unc'}) padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': reject, stats['p-corr'] = multicomp(stats['p-unc'].to_numpy(), method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None # Standardize correlation coefficients (Fisher z-transformation) stats['z'] = np.arctanh(stats['r'].to_numpy()) col_order = ['X', 'Y', 'method', 'tail', 'n', 'outliers', 'r', 'CI95%', 'r2', 'adj_r2', 'z', 'p-unc', 'p-corr', 'p-adjust', 'BF10', 'power'] # Reorder columns and remove empty ones stats = stats.reindex(columns=col_order).dropna(how='all', axis=1) # Add covariates names if present if covar is not None: stats.insert(loc=3, column='covar', value=str(covar)) return stats
def pairwise_ttests(data=None, dv=None, between=None, within=None, subject=None, parametric=True, marginal=True, alpha=.05, tail='two-sided', padjust='none', effsize='hedges', correction='auto', nan_policy='listwise', return_desc=False, interaction=True): """Pairwise T-tests. Parameters ---------- data : :py:class:`pandas.DataFrame` DataFrame. Note that this function can also directly be used as a Pandas method, in which case this argument is no longer needed. dv : string Name of column containing the dependant variable. between : string or list with 2 elements Name of column(s) containing the between-subject factor(s). .. warning:: Note that Pingouin gives slightly different T and p-values compared to JASP posthoc tests for 2-way factorial design, because Pingouin does not pool the standard error for each factor, but rather calculate each pairwise T-test completely independent of others. within : string or list with 2 elements Name of column(s) containing the within-subject factor(s), i.e. the repeated measurements. subject : string Name of column containing the subject identifier. This is compulsory when ``within`` is specified. parametric : boolean If True (default), use the parametric :py:func:`ttest` function. If False, use :py:func:`pingouin.wilcoxon` or :py:func:`pingouin.mwu` for paired or unpaired samples, respectively. marginal : boolean If True, average over repeated measures factor when working with mixed or two-way repeated measures design. For instance, in mixed design, the between-subject pairwise T-test(s) will be calculated after averaging across all levels of the within-subject repeated measures factor (the so-called *"marginal means"*). Similarly, in two-way repeated measures factor, the pairwise T-test(s) will be calculated after averaging across all levels of the other repeated measures factor. Setting ``marginal=True`` is recommended when doing posthoc testing with multiple factors in order to avoid violating the assumption of independence and conflating the degrees of freedom by the number of repeated measurements. This is the default behavior of JASP. .. warning:: The default behavior of Pingouin <0.3.2 was ``marginal = False``, which may have led to incorrect p-values for mixed or two-way repeated measures design. Make sure to always use the latest version of Pingouin. .. versionadded:: 0.3.2 alpha : float Significance level tail : string Specify whether the alternative hypothesis is `'two-sided'` or `'one-sided'`. Can also be `'greater'` or `'less'` to specify the direction of the test. `'greater'` tests the alternative that ``x`` has a larger mean than ``y``. If tail is `'one-sided'`, Pingouin will automatically infer the one-sided alternative hypothesis of the test based on the test statistic. padjust : string Method used for testing and adjustment of pvalues. * ``'none'``: no correction * ``'bonf'``: one-step Bonferroni correction * ``'sidak'``: one-step Sidak correction * ``'holm'``: step-down method using Bonferroni adjustments * ``'fdr_bh'``: Benjamini/Hochberg FDR correction * ``'fdr_by'``: Benjamini/Yekutieli FDR correction effsize : string or None Effect size type. Available methods are: * ``'none'``: no effect size * ``'cohen'``: Unbiased Cohen d * ``'hedges'``: Hedges g * ``'glass'``: Glass delta * ``'r'``: Pearson correlation coefficient * ``'eta-square'``: Eta-square * ``'odds-ratio'``: Odds ratio * ``'AUC'``: Area Under the Curve * ``'CLES'``: Common Language Effect Size correction : string or boolean For unpaired two sample T-tests, specify whether or not to correct for unequal variances using Welch separate variances T-test. If `'auto'`, it will automatically uses Welch T-test when the sample sizes are unequal, as recommended by Zimmerman 2004. .. versionadded:: 0.3.2 nan_policy : string Can be `'listwise'` for listwise deletion of missing values in repeated measures design (= complete-case analysis) or `'pairwise'` for the more liberal pairwise deletion (= available-case analysis). .. versionadded:: 0.2.9 return_desc : boolean If True, append group means and std to the output dataframe interaction : boolean If there are multiple factors and ``interaction`` is True (default), Pingouin will also calculate T-tests for the interaction term (see Notes). .. versionadded:: 0.2.9 Returns ------- stats : :py:class:`pandas.DataFrame` * ``'A'``: Name of first measurement * ``'B'``: Name of second measurement * ``'Paired'``: indicates whether the two measurements are paired or not * ``'Parametric'``: indicates if (non)-parametric tests were used * ``'Tail'``: indicate whether the p-values are one-sided or two-sided * ``'T'``: T statistic (only if parametric=True) * ``'U-val'``: Mann-Whitney U stat (if parametric=False and unpaired data) * ``'W-val'``: Wilcoxon W stat (if parametric=False and paired data) * ``'dof'``: degrees of freedom (only if parametric=True) * ``'p-unc'``: Uncorrected p-values * ``'p-corr'``: Corrected p-values * ``'p-adjust'``: p-values correction method * ``'BF10'``: Bayes Factor * ``'hedges'``: effect size (or any effect size defined in ``effsize``) See also -------- ttest, mwu, wilcoxon, compute_effsize, multicomp Notes ----- Data are expected to be in long-format. If your data is in wide-format, you can use the :py:func:`pandas.melt` function to convert from wide to long format. If ``between`` or ``within`` is a list (e.g. ['col1', 'col2']), the function returns 1) the pairwise T-tests between each values of the first column, 2) the pairwise T-tests between each values of the second column and 3) the interaction between col1 and col2. The interaction is dependent of the order of the list, so ['col1', 'col2'] will not yield the same results as ['col2', 'col1'], and will only be calculated if ``interaction=True``. In other words, if ``between`` is a list with two elements, the output model is between1 + between2 + between1 * between2. Similarly, if ``within`` is a list with two elements, the output model is within1 + within2 + within1 * within2. If both ``between`` and ``within`` are specified, the output model is within + between + within * between (= mixed design). Missing values in repeated measurements are automatically removed using a listwise (default) or pairwise deletion strategy. However, you should be very careful since it can result in undesired values removal (especially for the interaction effect). We strongly recommend that you preprocess your data and remove the missing values before using this function. This function has been tested against the `pairwise.t.test <https://www.rdocumentation.org/packages/stats/versions/3.6.2/topics/pairwise.t.test>`_ R function. .. warning:: Versions of Pingouin below 0.3.2 gave incorrect results for mixed and two-way repeated measures design (see above warning for the ``marginal`` argument). .. warning:: Pingouin gives slightly different results than the JASP's posthoc module when working with multiple factors (e.g. mixed, factorial or 2-way repeated measures design). This is mostly caused by the fact that Pingouin does not pool the standard error for between-subject and interaction contrasts. You should always double check your results with JASP or another statistical software. Examples -------- For more examples, please refer to the `Jupyter notebooks <https://github.com/raphaelvallat/pingouin/blob/master/notebooks/01_ANOVA.ipynb>`_ 1. One between-subject factor >>> from pingouin import pairwise_ttests, read_dataset >>> df = read_dataset('mixed_anova.csv') >>> pairwise_ttests(dv='Scores', between='Group', data=df) # doctest: +SKIP 2. One within-subject factor >>> post_hocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', data=df) >>> print(post_hocs) # doctest: +SKIP 3. Non-parametric pairwise paired test (wilcoxon) >>> pairwise_ttests(dv='Scores', within='Time', subject='Subject', ... data=df, parametric=False) # doctest: +SKIP 4. Mixed design (within and between) with bonferroni-corrected p-values >>> posthocs = pairwise_ttests(dv='Scores', within='Time', ... subject='Subject', between='Group', ... padjust='bonf', data=df) 5. Two between-subject factors. The order of the list matters! >>> posthocs = pairwise_ttests(dv='Scores', between=['Group', 'Time'], ... data=df) 6. Same but without the interaction >>> posthocs = df.pairwise_ttests(dv='Scores', between=['Group', 'Time'], ... interaction=False) """ from .parametric import ttest from .nonparametric import wilcoxon, mwu # Safety checks _check_dataframe(dv=dv, between=between, within=within, subject=subject, effects='all', data=data) assert tail in ['one-sided', 'two-sided', 'greater', 'less'] assert isinstance(alpha, float), 'alpha must be float.' assert nan_policy in ['listwise', 'pairwise'] # Check if we have multiple between or within factors multiple_between = False multiple_within = False contrast = None if isinstance(between, list): if len(between) > 1: multiple_between = True contrast = 'multiple_between' assert all([b in data.keys() for b in between]) else: between = between[0] if isinstance(within, list): if len(within) > 1: multiple_within = True contrast = 'multiple_within' assert all([w in data.keys() for w in within]) else: within = within[0] if all([multiple_within, multiple_between]): raise ValueError("Multiple between and within factors are", "currently not supported. Please select only one.") # Check the other cases if isinstance(between, str) and within is None: contrast = 'simple_between' assert between in data.keys() if isinstance(within, str) and between is None: contrast = 'simple_within' assert within in data.keys() if isinstance(between, str) and isinstance(within, str): contrast = 'within_between' assert all([between in data.keys(), within in data.keys()]) # Reorganize column order col_order = ['Contrast', 'Time', 'A', 'B', 'mean(A)', 'std(A)', 'mean(B)', 'std(B)', 'Paired', 'Parametric', 'T', 'U-val', 'W-val', 'dof', 'Tail', 'p-unc', 'p-corr', 'p-adjust', 'BF10', effsize] if contrast in ['simple_within', 'simple_between']: # OPTION A: SIMPLE MAIN EFFECTS, WITHIN OR BETWEEN paired = True if contrast == 'simple_within' else False col = within if contrast == 'simple_within' else between # Remove NAN in repeated measurements if contrast == 'simple_within' and data[dv].isnull().to_numpy().any(): # Only if nan_policy == 'listwise'. For pairwise deletion, # missing values will be removed directly in the lower-level # functions (e.g. pg.ttest) if nan_policy == 'listwise': data = remove_rm_na(dv=dv, within=within, subject=subject, data=data) else: # The `remove_rm_na` also aggregate other repeated measures # factor using the mean. Here, we ensure this behavior too. data = data.groupby([subject, within])[dv].mean().reset_index() # Now we check that subjects are present in all conditions # For example, if we have four subjects and 3 conditions, # and if subject 2 have missing data at the third condition, # we still need a row with missing values for this subject. if data.groupby(within)[subject].count().nunique() != 1: raise ValueError("Repeated measures dataframe is not balanced." " `Subjects` must have the same number of " "elements in all conditions, " "even when missing values are present.") # Extract effects grp_col = data.groupby(col, sort=False)[dv] labels = grp_col.groups.keys() # Number and labels of possible comparisons if len(labels) >= 2: combs = list(combinations(labels, 2)) combs = np.array(combs) A = combs[:, 0] B = combs[:, 1] else: raise ValueError('Columns must have at least two unique values.') # Initialize dataframe stats = pd.DataFrame(dtype=np.float64, index=range(len(combs)), columns=col_order) # Force dtype conversion cols_str = ['Contrast', 'Time', 'A', 'B', 'Tail', 'p-adjust', 'BF10'] cols_bool = ['Parametric', 'Paired'] stats[cols_str] = stats[cols_str].astype(object) stats[cols_bool] = stats[cols_bool].astype(bool) # Fill str columns stats.loc[:, 'A'] = A stats.loc[:, 'B'] = B stats.loc[:, 'Contrast'] = col stats.loc[:, 'Tail'] = tail stats.loc[:, 'Paired'] = paired for i in range(stats.shape[0]): col1, col2 = stats.at[i, 'A'], stats.at[i, 'B'] x = grp_col.get_group(col1).to_numpy(dtype=np.float64) y = grp_col.get_group(col2).to_numpy(dtype=np.float64) if parametric: stat_name = 'T' df_ttest = ttest(x, y, paired=paired, tail=tail, correction=correction) stats.at[i, 'BF10'] = df_ttest.at['T-test', 'BF10'] stats.at[i, 'dof'] = df_ttest.at['T-test', 'dof'] else: if paired: stat_name = 'W-val' df_ttest = wilcoxon(x, y, tail=tail) else: stat_name = 'U-val' df_ttest = mwu(x, y, tail=tail) # Compute Hedges / Cohen ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) if return_desc: stats.at[i, 'mean(A)'] = np.nanmean(x) stats.at[i, 'mean(B)'] = np.nanmean(y) stats.at[i, 'std(A)'] = np.nanstd(x, ddof=1) stats.at[i, 'std(B)'] = np.nanstd(y, ddof=1) stats.at[i, stat_name] = df_ttest[stat_name].iat[0] stats.at[i, 'p-unc'] = df_ttest['p-val'].iat[0] stats.at[i, effsize] = ef # Multiple comparisons padjust = None if stats['p-unc'].size <= 1 else padjust if padjust is not None: if padjust.lower() != 'none': _, stats['p-corr'] = multicomp(stats['p-unc'].to_numpy(), alpha=alpha, method=padjust) stats['p-adjust'] = padjust else: stats['p-corr'] = None stats['p-adjust'] = None else: # Multiple factors if contrast == 'multiple_between': # B1: BETWEEN1 + BETWEEN2 + BETWEEN1 * BETWEEN2 factors = between fbt = factors fwt = [None, None] paired = False # the interaction is not paired agg = [False, False] # TODO: add a pool SD option, as in JASP and JAMOVI? elif contrast == 'multiple_within': # B2: WITHIN1 + WITHIN2 + WITHIN1 * WITHIN2 factors = within fbt = [None, None] fwt = factors paired = True agg = [True, True] # Calculate marginal means for both factors else: # B3: WITHIN + BETWEEN + WITHIN * BETWEEN factors = [within, between] fbt = [None, between] fwt = [within, None] paired = False agg = [False, True] stats = pd.DataFrame() for i, f in enumerate(factors): # Introduced in Pingouin v0.3.2 if all([agg[i], marginal]): tmp = data.groupby([subject, f], as_index=False, sort=False).mean() else: tmp = data stats = stats.append(pairwise_ttests(dv=dv, between=fbt[i], within=fwt[i], subject=subject, data=tmp, parametric=parametric, marginal=marginal, alpha=alpha, tail=tail, padjust=padjust, effsize=effsize, correction=correction, nan_policy=nan_policy, return_desc=return_desc), ignore_index=True, sort=False) # Then compute the interaction between the factors if interaction: nrows = stats.shape[0] grp_fac1 = data.groupby(factors[0], sort=False)[dv] grp_fac2 = data.groupby(factors[1], sort=False)[dv] grp_both = data.groupby(factors, sort=False)[dv] labels_fac1 = grp_fac1.groups.keys() labels_fac2 = grp_fac2.groups.keys() # comb_fac1 = list(combinations(labels_fac1, 2)) comb_fac2 = list(combinations(labels_fac2, 2)) # Pairwise comparisons combs_list = list(product(labels_fac1, comb_fac2)) ncombs = len(combs_list) # np.array(combs_list) does not work because of tuples # we therefore need to flatten the tupple combs = np.zeros(shape=(ncombs, 3), dtype=object) for i in range(ncombs): combs[i] = _flatten_list(combs_list[i], include_tuple=True) # Append empty rows idxiter = np.arange(nrows, nrows + ncombs) stats = stats.append(pd.DataFrame(columns=stats.columns, index=idxiter), ignore_index=True) # Update other columns stats.loc[idxiter, 'Contrast'] = factors[0] + ' * ' + factors[1] stats.loc[idxiter, 'Time'] = combs[:, 0] stats.loc[idxiter, 'Paired'] = paired stats.loc[idxiter, 'Tail'] = tail stats.loc[idxiter, 'A'] = combs[:, 1] stats.loc[idxiter, 'B'] = combs[:, 2] for i, comb in enumerate(combs): ic = nrows + i # Take into account previous rows fac1, col1, col2 = comb x = grp_both.get_group((fac1, col1)).to_numpy(dtype=np.float64) y = grp_both.get_group((fac1, col2)).to_numpy(dtype=np.float64) ef = compute_effsize(x=x, y=y, eftype=effsize, paired=paired) if parametric: stat_name = 'T' df_ttest = ttest(x, y, paired=paired, tail=tail, correction=correction) stats.at[ic, 'BF10'] = df_ttest.at['T-test', 'BF10'] stats.at[ic, 'dof'] = df_ttest.at['T-test', 'dof'] else: if paired: stat_name = 'W-val' df_ttest = wilcoxon(x, y, tail=tail) else: stat_name = 'U-val' df_ttest = mwu(x, y, tail=tail) # Append to stats if return_desc: stats.at[ic, 'mean(A)'] = np.nanmean(x) stats.at[ic, 'mean(B)'] = np.nanmean(y) stats.at[ic, 'std(A)'] = np.nanstd(x, ddof=1) stats.at[ic, 'std(B)'] = np.nanstd(y, ddof=1) stats.at[ic, stat_name] = df_ttest[stat_name].iat[0] stats.at[ic, 'p-unc'] = df_ttest['p-val'].iat[0] stats.at[ic, effsize] = ef # Multi-comparison columns if padjust is not None and padjust.lower() != 'none': _, pcor = multicomp(stats.loc[idxiter, 'p-unc'].to_numpy(), alpha=alpha, method=padjust) stats.loc[idxiter, 'p-corr'] = pcor stats.loc[idxiter, 'p-adjust'] = padjust # --------------------------------------------------------------------- # Append parametric columns stats.loc[:, 'Parametric'] = parametric # Reorder and drop empty columns stats = stats[np.array(col_order)[np.isin(col_order, stats.columns)]] stats = stats.dropna(how='all', axis=1) # Rename Time columns if (contrast in ['multiple_within', 'multiple_between', 'within_between'] and interaction): stats['Time'].fillna('-', inplace=True) stats.rename(columns={'Time': factors[0]}, inplace=True) return stats