def between_correls(args):
"""TABLES MUST SORT SO THAT SAMPLES ARE IN THE SAME ORDER """
logger = general.Logger("SCNIC_log.txt")
logger["SCNIC analysis type"] = "between"
# correlation and p-value adjustment methods
correl_methods = {'spearman': spearmanr, 'pearson': pearsonr}
p_methods = {'bh': general.bh_adjust, 'bon': general.bonferroni_adjust}
correl_method = correl_methods[args.correl_method]
if args.p_adjust is not None:
p_adjust = p_methods[args.p_adjust]
else:
p_adjust = None
# load tables
table1 = load_table(args.table1)
table2 = load_table(args.table2)
logger["input table 1"] = args.table1
logger["input table 1"] = args.table2
table1 = table1.sort()
table2 = table2.sort()
if not np.array_equal(table1.ids(), table2.ids()):
raise ValueError("Tables have different sets of samples present")
# make new output directory and change to it
if args.output is not None:
os.makedirs(args.output)
os.chdir(args.output)
logger["output directory"] = args.output
# filter tables
if args.min_sample is not None:
table1 = general.filter_table(table1, args.min_sample)
metadata = general.get_metadata_from_table(table1)
table2 = general.filter_table(table2, args.min_sample)
metadata.update(general.get_metadata_from_table(table2))
else:
metadata = general.get_metadata_from_table(table1)
metadata.update(general.get_metadata_from_table(table2))
# make correlations
logger["correlation metric"] = args.correl_method
logger["p adjustment method"] = args.p_adjust
correls = between_correls_from_tables(table1, table2, correl_method)
correls.sort_values(correls.columns[-1], inplace=True)
correls.to_csv(open('correls.txt', 'w'), sep='\t', index=False)
# adjust p-values
correls['p_adj'] = p_adjust(correls['p'])
# make network
net = general.correls_to_net(correls, metadata=metadata, min_p=args.min_p, min_r=args.min_r)
logger["number of nodes"] = net.number_of_nodes()
logger["number of edges"] = net.number_of_edges()
nx.write_gml(net, 'crossnet.gml')
logger.output_log()
print '\a'
def test_filter_better(biom_table2):
table_filt = filter_table(biom_table2, min_samples=4)
assert len(table_filt.ids(axis="observation")) == 4
assert len(table_filt.ids(axis="sample")) == 5
def test_filter_table(biom_table1):
table_filt = filter_table(biom_table1, min_samples=10)
assert isinstance(table_filt, Table)
def between_correls(args):
"""TABLES MUST SORT SO THAT SAMPLES ARE IN THE SAME ORDER """
logger = general.Logger("SCNIC_log.txt")
logger["SCNIC analysis type"] = "between"
# correlation and p-value adjustment methods
correl_methods = {'spearman': spearmanr, 'pearson': pearsonr}
correl_method = correl_methods[args.correl_method]
# load tables
table1 = load_table(args.table1)
table2 = load_table(args.table2)
logger["input table 1"] = args.table1
logger["input table 1"] = args.table2
table1 = table1.sort()
table2 = table2.sort()
# make new output directory and change to it
if args.force and args.output is not None:
shutil.rmtree(args.output, ignore_errors=True)
if args.output is not None:
os.makedirs(args.output)
os.chdir(args.output)
logger["output directory"] = args.output
# filter tables
if args.sparcc_filter is True:
table1 = general.sparcc_paper_filter(table1)
table2 = general.sparcc_paper_filter(table2)
print("Table 1 filtered: %s observations" % str(table1.shape[0]))
print("Table 2 filtered: %s observations" % str(table2.shape[0]))
logger["sparcc paper filter"] = True
logger["number of observations present in table 1 after filter"] = table1.shape[0]
logger["number of observations present in table 2 after filter"] = table2.shape[0]
if args.min_sample is not None:
table1 = general.filter_table(table1, args.min_sample)
table2 = general.filter_table(table2, args.min_sample)
if not np.array_equal(table1.ids(), table2.ids()):
raise ValueError("Tables have different sets of samples present")
metadata = general.get_metadata_from_table(table1)
metadata.update(general.get_metadata_from_table(table2))
# make correlations
logger["correlation metric"] = args.correl_method
logger["p adjustment method"] = args.p_adjust
correls = ca.between_correls_from_tables(table1, table2, correl_method, nprocs=args.procs)
correls.sort_values(correls.columns[-1], inplace=True)
correls['p_adj'] = general.p_adjust(correls['p'])
correls.to_csv(open('correls.txt', 'w'), sep='\t', index=True)
# make network
correls_filt = general.filter_correls(correls, min_p=args.min_p, min_r=args.min_r)
net = general.correls_to_net(correls_filt, metadata=metadata)
logger["number of nodes"] = net.number_of_nodes()
logger["number of edges"] = net.number_of_edges()
nx.write_gml(net, 'crossnet.gml')
logger.output_log()
def within_correls(input_loc, output_loc, correl_method='sparcc', sparcc_filter=False, min_sample=None, procs=1,
sparcc_p=1000, p_adjust='fdr_bh', verbose=False):
logger = general.Logger(path.join(output_loc, "SCNIC_within_log.txt"))
logger["SCNIC analysis type"] = "within"
# correlation and p-value adjustment methods
correl_methods = {'spearman': spearmanr, 'pearson': pearsonr, 'kendall': kendalltau, 'sparcc': 'sparcc'}
correl_method = correl_methods[correl_method.lower()]
# get features to be correlated
table = load_table(input_loc)
logger["input table"] = input_loc
if verbose:
print("Table loaded: " + str(table.shape[0]) + " observations")
print("")
logger["number of samples in input table"] = table.shape[1]
logger["number of observations in input table"] = table.shape[0]
# make new output directory
if output_loc is not None:
if not path.isdir(output_loc):
os.makedirs(output_loc)
logger["output directory"] = path.abspath(output_loc)
# filter
if sparcc_filter is True:
table_filt = general.sparcc_paper_filter(table)
if verbose:
print("Table filtered: %s observations" % str(table_filt.shape[0]))
print("")
logger["sparcc paper filter"] = True
logger["number of observations present after filter"] = table_filt.shape[0]
elif min_sample is not None:
table_filt = general.filter_table(table, min_sample)
if verbose:
print("Table filtered: %s observations" % str(table_filt.shape[0]))
print("")
logger["min samples present"] = min_sample
logger["number of observations present after filter"] = table_filt.shape[0]
else:
table_filt = table
logger["number of processors used"] = procs
# correlate features
if correl_method in [spearmanr, pearsonr, kendalltau]:
# calculate correlations
if verbose:
print("Correlating with %s" % correl_method)
# correlate feature
correls = ca.calculate_correlations(table_filt, correl_method, nprocs=procs, p_adjust_method=p_adjust)
elif correl_method == 'sparcc':
if sparcc_p is None:
correls = ca.fastspar_correlation(table_filt, verbose=verbose, nprocs=procs)
else:
correls = ca.fastspar_correlation(table_filt, calc_pvalues=True, bootstraps=sparcc_p,
verbose=verbose, nprocs=procs, p_adjust_method=p_adjust)
else:
raise ValueError("How did this even happen?")
logger["distance metric used"] = correl_method
if verbose:
print("Features Correlated")
print("")
correls.to_csv(path.join(output_loc, 'correls.txt'), sep='\t', index_label=('feature1', 'feature2'))
if verbose:
print("Correls.txt written")
# make correlation network
metadata = general.get_metadata_from_table(table_filt)
net = general.correls_to_net(correls, metadata=metadata)
nx.write_gml(net, path.join(output_loc, 'correlation_network.gml'))
if verbose:
print("Network made")
print("")
logger.output_log()
def within_correls(args):
logger = general.Logger("SCNIC_within_log.txt")
logger["SCNIC analysis type"] = "within"
# correlation and p-value adjustment methods
correl_methods = {'spearman': spearmanr, 'pearson': pearsonr, 'kendall': kendalltau, 'sparcc': 'sparcc'}
correl_method = correl_methods[args.correl_method.lower()]
# get features to be correlated
table = load_table(args.input)
logger["input table"] = args.input
if args.verbose:
print("Table loaded: " + str(table.shape[0]) + " observations")
print("")
logger["number of samples in input table"] = table.shape[1]
logger["number of observations in input table"] = table.shape[0]
# make new output directory and change to it
if args.output is not None:
if not os.path.isdir(args.output):
os.makedirs(args.output)
os.chdir(args.output)
logger["output directory"] = os.getcwd()
# filter
if args.sparcc_filter is True:
table_filt = general.sparcc_paper_filter(table)
if args.verbose:
print("Table filtered: %s observations" % str(table_filt.shape[0]))
print("")
logger["sparcc paper filter"] = True
logger["number of observations present after filter"] = table_filt.shape[0]
elif args.min_sample is not None:
table_filt = general.filter_table(table, args.min_sample)
if args.verbose:
print("Table filtered: %s observations" % str(table_filt.shape[0]))
print("")
logger["min samples present"] = args.min_sample
logger["number of observations present after filter"] = table_filt.shape[0]
else:
table_filt = table
logger["number of processors used"] = args.procs
# correlate features
if correl_method in [spearmanr, pearsonr, kendalltau]:
# calculate correlations
if args.verbose:
print("Correlating with %s" % args.correl_method)
# correlate feature
correls = ca.calculate_correlations(table_filt, correl_method)
elif correl_method == 'sparcc':
correls = ca.fastspar_correlation(table_filt, verbose=args.verbose)
if args.sparcc_p is not None:
raise NotImplementedError() # TODO: reimplement with fastspar
else:
raise ValueError("How did this even happen?")
logger["distance metric used"] = args.correl_method
if args.verbose:
print("Features Correlated")
print("")
if 'p' in correls.columns:
correls['p_adj'] = general.p_adjust(correls['p'])
correls.to_csv('correls.txt', sep='\t', index_label=('feature1', 'feature2'))
if args.verbose:
print("Correls.txt written")
# make correlation network
metadata = general.get_metadata_from_table(table_filt)
net = general.correls_to_net(correls, metadata=metadata)
nx.write_gml(net, 'correlation_network.gml')
if args.verbose:
print("Network made")
print("")
logger.output_log()
def test_filter_table(biom_table1):
table_filt = filter_table(biom_table1, min_samples=10)
assert type(table_filt) is Table