def main(counts, method='SparCC', **kwargs):
'''
Compute correlations between all components of counts matrix.
Run several iterations, in each the fractions are re-estimated,
and return the median of all iterations.
Running several iterations is only helpful with 'dirichlet'
normalization method, as with other methods all iterations
will give identical results. Thus, if using other normalizations
set 'iter' parameter to 1.
Parameters
----------
counts : DataFrame
2D array of counts. Columns are components, rows are samples.
If using 'dirichlet' or 'pseudo' normalization,
counts (positive integers) are required to produce meaningful results,
though this is not explicitly checked by the code.
method : str, optional (default 'SparCC')
The algorithm to use for computing correlation.
Supported values: SparCC, clr, pearson, spearman, kendall
Note that the pearson, spearman, kendall methods are not
altered to account for the fact that the data is compositional,
and are provided to facilitate comparisons to
the clr and sparcc methods.
Returns
-------
cor_med: array
Estimated correlation values.
cov_med: array
Estimated covariance matrix if method in {SparCC, clr},
None otherwise.
======= ============ ======= ================================================
kwarg Accepts Default Desctiption
======= ============ ======= ================================================
iter int 20 number of estimation iteration to average over.
oprint bool True print iteration progress?
th 0<th<1 0.1 exclusion threshold for SparCC.
xiter int 10 number of exclusion iterations for sparcc.
norm str dirichlet method used to normalize the counts to fractions.
log bool True log-transform fraction? used if method ~= SparCC/CLR
======= ============ ========= ================================================
'''
method = method.lower()
cor_list = [] # list of cor matrices from different random fractions
var_list = [] # list of cov matrices from different random fractions
oprint = kwargs.pop('oprint', True)
n_iter = kwargs.pop('iter', 20) # number of iterations
norm = kwargs.pop('norm', 'dirichlet')
log = kwargs.pop('log', 'True')
th = kwargs.setdefault(
'th', 0.1) # exclusion threshold for iterative sparse algo
if method in ['sparcc', 'clr']:
for i in range(n_iter):
if oprint: print '\tRunning iteration' + str(i)
fracs = to_fractions(counts, method=norm)
v_sparse, cor_sparse, cov_sparse = basis_corr(fracs,
method=method,
**kwargs)
var_list.append(np.diag(cov_sparse))
cor_list.append(cor_sparse)
cor_array = np.array(cor_list)
var_med = nanmedian(var_list, axis=0) #median variances
cor_med = nanmedian(cor_array, axis=0) #median correlations
x, y = np.meshgrid(var_med, var_med)
cov_med = cor_med * x**0.5 * y**0.5
elif method in ['pearson', 'kendall', 'spearman']:
n = counts.shape[1]
cor_array = np.zeros((n_iter, n, n))
for i in range(n_iter):
if oprint: print '\tRunning iteration ' + str(i)
fracs = to_fractions(counts, method=norm)
if log:
x = np.log(fracs)
else:
x = fracs
cor_mat, pval = correlation(x, method, axis=0)
cor_array[i, :, :] = cor_mat
cor_med = np.median(cor_array, axis=0) #median correlation
cov_med = None
return cor_med, cov_med
def driver():
#if __name__ == '__main__':
#print "RUNNING CODE MAIN"
## parse input arguments
from optparse import OptionParser
kwargs = {}
usage = (
'Compute the correlation between components (e.g. OTUs).\n'
'By default uses the SparCC algorithm to account for compositional effects.\n'
'Correlation and covariance (when applies) matrices are written out as txt files. \n'
'Counts file needs to be a tab delimited text file where columns are samples and rows are components (e.g. OTUS).\n'
' See example/fake_data.txt for an example file.\n'
'\n'
'Usage: python SparCC.py counts_file [options]\n'
'Example: python SparCC.py example/fake_data.txt -i 20 --cor_file=example/basis_corr/cor_mat_sparcc.out'
)
parser = OptionParser(usage)
parser.add_option("-c",
"--cor_file",
dest="cor_file",
type='str',
help="File to which correlation matrix will be written.")
parser.add_option("-v",
"--cov_file",
dest="cov_file",
type='str',
help="File to which covariance matrix will be written.")
parser.add_option(
"-a",
"--algo",
dest="algo",
default='SparCC',
help=
"Name of algorithm used to compute correlations (SparCC (default) | pearson | spearman | kendall)"
)
parser.add_option(
"-i",
"--iter",
dest='iter',
type='int',
default=20,
help="Number of inference iterations to average over (20 default).")
parser.add_option(
"-x",
"--xiter",
dest='xiter',
type='int',
default=10,
help=
"Number of exclusion iterations to remove strongly correlated pairs (10 default)."
)
parser.add_option(
"-t",
"--thershold",
dest='th',
type='float',
default=0.1,
help="Correlation strength exclusion threshold (0.1 default).")
parser.add_option("-p",
"--pval_file",
dest="pval_file",
type='str',
help="File to which pvalues will be written.")
(options, args) = parser.parse_args()
#print "OPTIONS: ", options
#print "ARGS: ", args
counts_file = args[0]
from analysis_methods import basis_corr
from io_methods import read_txt, write_txt
kwargs = options.__dict__
algo = kwargs.pop('algo')
cor_file = kwargs.pop('cor_file')
cov_file = kwargs.pop('cov_file')
pval_file = kwargs.pop('pval_file')
if cor_file is None: cor_file = 'cor_mat_' + algo + '.out'
if cov_file is None: cov_file = 'cov_mat_' + algo + '.out'
if pval_file is None: pval_file = 'pval_mat_' + algo + '.out'
print('reading data')
counts = read_txt(counts_file)
## Calculate correlations between components using SparCC
print('computing correlations')
cor, cov, pval = basis_corr(counts, method=algo, **kwargs)
print(counts)
## write out results
print('writing results')
write_txt(cor, cor_file)
print('wrote ' + cor_file)
if cov is not None:
write_txt(cov, cov_file)
#print 'wrote ' + cov_file
if pval is not None:
write_txt(pval, pval_file)
help="Correlation strength exclusion threshold (0.1 default).")
(options, args) = parser.parse_args()
counts_file = args[0]
from analysis_methods import basis_corr
from io_methods import read_txt, write_txt
kwargs = options.__dict__
algo = kwargs.pop('algo')
cor_file = kwargs.pop('cor_file')
cov_file = kwargs.pop('cov_file')
if cor_file is None: cor_file = 'cor_mat_' + algo + '.out'
if cov_file is None: cov_file = 'cov_mat_' + algo + '.out'
print 'reading data'
counts = read_txt(counts_file)
## Calculate correlations between components using SparCC
print 'computing correlations'
cor, cov = basis_corr(counts, method=algo, **kwargs)
## write out results
print 'writing results'
write_txt(cor, cor_file)
print 'wrote ' + cor_file
if cov is not None:
write_txt(cov, cov_file)
print 'wrote ' + cov_file
print 'Done!'
def main(counts, method='sparcc', **kwargs):
'''
Compute correlations between all components of counts matrix.
Run several iterations, in each the fractions are re-estimated,
and return the median of all iterations.
Running several iterations is only helpful with 'dirichlet'
normalization method, as with other methods all iterations
will give identical results. Thus, if using other normalizations
set 'iter' parameter to 1.
Parameters
----------
counts : DataFrame
2D array of counts. Columns are components, rows are samples.
If using 'dirichlet' or 'pseudo' normalization,
counts (positive integers) are required to produce meaningful results,
though this is not explicitly checked by the code.
method : str, optional (default 'SparCC')
The algorithm to use for computing correlation.
Supported values: SparCC, clr, pearson, spearman, kendall
Note that the pearson, spearman, kendall methods are not
altered to account for the fact that the data is compositional,
and are provided to facilitate comparisons to
the clr and sparcc methods.
Returns
-------
cor_med: array
Estimated correlation values.
cov_med: array
Estimated covariance matrix if method in {SparCC, clr},
None otherwise.
======= ============ ======= ================================================
kwarg Accepts Default Desctiption
======= ============ ======= ================================================
iter int 20 number of estimation iteration to average over.
oprint bool True print iteration progress?
th 0<th<1 0.1 exclusion threshold for SparCC.
xiter int 10 number of exclusion iterations for sparcc.
norm str dirichlet method used to normalize the counts to fractions.
log bool True log-transform fraction? used if method ~= SparCC/CLR
======= ============ ========= ================================================
'''
#print "RUNNING FUNCTION MAIN"
method = method.lower()
cor_list = [] # list of cor matrices from different random fractions
var_list = [] # list of cov matrices from different random fractions
oprint = kwargs.pop('oprint', True)
n_iter = kwargs.pop('iter', 20) # number of iterations
norm = kwargs.pop('norm', 'normalize') #'dirichlet')
log = kwargs.pop('log', 'True')
th = kwargs.setdefault(
'th', 0.1) # exclusion threshold for iterative sparse algo
if method in ['sparcc', 'clr']:
for i in range(n_iter):
if oprint: print('\tRunning iteration' + str(i))
#fracs = to_fractions(counts, method=norm)
fracs = counts
# PUT TO LOW VALUES?
for col in fracs.keys():
for otu in fracs[col].keys():
if (fracs[col][otu] == 0):
fracs[col][otu] = 1e-14
#print "FRACS: ", fracs
v_sparse, cor_sparse, cov_sparse = basis_corr(fracs,
method=method,
**kwargs)
#print "V SPARSE: ", v_sparse
#print "COR SPARSE: ", cor_sparse
#print "COV SPARSE: ", cov_sparse
var_list.append(np.diag(cov_sparse))
cor_list.append(cor_sparse)
cor_array = np.array(cor_list)
#print "VAR MED"
#print "VARLIST: ", var_list
var_med = nanmedian(var_list, axis=0) #median variances
#print "COR MED"
cor_med = nanmedian(cor_array, axis=0) #median correlations
#print "MESH GRID"
x, y = np.meshgrid(var_med, var_med)
#print "COV MED"
cov_med = cor_med * x**0.5 * y**0.5
pval = None
elif method in ['pearson', 'kendall', 'spearman']:
n = counts.shape[1]
cor_array = np.zeros((n_iter, n, n))
for i in range(n_iter):
if oprint: print('\tRunning iteration ' + str(i))
fracs = counts
# Attempting this scaling in Pearson/Spearman as well
#for col in fracs.keys():
# for otu in fracs[col].keys():
# if (fracs[col][otu] == 0):
# fracs[col][otu] = random.randint(1,9)*math.pow(10,-14)
#print "SET TO: ", fracs[col][otu]
#fracs = to_fractions(counts, method=norm)
#if log:
# x = np.log(fracs)
#else:
# x = fracs
x = fracs
cor_mat, pval = correlation(x, method, axis=0)
#print "COR_MAT: ", cor_mat
#print "PVAL: ", pval
cor_array[i, :, :] = cor_mat
cor_med = np.median(cor_array, axis=0) #median correlation
#print "COR_MED: ", cor_med
cov_med = None
return cor_med, cov_med, pval