def test_from_file_with_file_path(self):
"""Should identify the filepath correctly and parse from it."""
# should fail with the expected exception
with self.assertRaises(DissimilarityMatrixFormatError):
_ = DistanceMatrix.from_file(self.bad_dm_fp)
obs = DistanceMatrix.from_file(self.dm_3x3_fp)
self.assertEqual(self.dm_3x3, obs)
self.assertTrue(isinstance(obs, DistanceMatrix))
def test_from_file_with_file_path(self):
"""Should identify the filepath correctly and parse from it."""
# should fail with the expected exception
with self.assertRaises(DissimilarityMatrixFormatError):
DistanceMatrix.from_file(self.bad_dm_fp)
obs = DistanceMatrix.from_file(self.dm_3x3_fp)
self.assertEqual(self.dm_3x3, obs)
self.assertTrue(isinstance(obs, DistanceMatrix))
def pcoa(lines):
"""Run PCoA on the distance matrix present on lines"""
# Parse the distance matrix
dist_mtx = DistanceMatrix.from_file(lines)
# Create the PCoA object
pcoa_obj = PCoA(dist_mtx)
# Get the PCoA results and return them
return pcoa_obj.scores()
def pcoa(lines):
"""Run PCoA on the distance matrix present on lines"""
# Parse the distance matrix
dist_mtx = DistanceMatrix.from_file(lines)
# Create the PCoA object
pcoa_obj = PCoA(dist_mtx)
# Get the PCoA results and return them
return pcoa_obj.scores()
def setup(self):
with open(get_data_path('PCoA_sample_data_3'), 'U') as lines:
dist_matrix = DistanceMatrix.from_file(lines)
self.ordination = PCoA(dist_matrix)
self.ids = [
'PC.636', 'PC.635', 'PC.356', 'PC.481', 'PC.354', 'PC.593',
'PC.355', 'PC.607', 'PC.634'
]
def single_file_nj(input_file, output_file):
dm = DistanceMatrix.from_file(input_file)
tree = nj(dm)
# write output
f = open(output_file, 'w')
f.write(tree.to_newick(with_distances=True))
f.close()
def single_file_upgma(input_file, output_file):
# read in dist matrix
dist_mat = DistanceMatrix.from_file(input_file)
# SciPy uses average as UPGMA:
# http://docs.scipy.org/doc/scipy/reference/generated/
# scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
linkage_matrix = linkage(dist_mat.condensed_form(), method='average')
tree = TreeNode.from_linkage_matrix(linkage_matrix, dist_mat.ids)
# write output
f = open(output_file, 'w')
try:
f.write(tree.to_newick(with_distances=True))
except AttributeError:
if c is None:
raise RuntimeError("""input file %s did not make a UPGMA tree.
Ensure it has more than one sample present""" % (str(input_file), ))
raise
f.close()
def single_file_upgma(input_file, output_file):
# read in dist matrix
dist_mat = DistanceMatrix.from_file(input_file)
# SciPy uses average as UPGMA:
# http://docs.scipy.org/doc/scipy/reference/generated/
# scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
linkage_matrix = linkage(dist_mat.condensed_form(), method='average')
tree = TreeNode.from_linkage_matrix(linkage_matrix, dist_mat.ids)
# write output
f = open(output_file, 'w')
try:
f.write(tree.to_newick(with_distances=True))
except AttributeError:
if c is None:
raise RuntimeError("""input file %s did not make a UPGMA tree.
Ensure it has more than one sample present""" % (str(input_file),))
raise
f.close()
def test_from_file_invalid_input(self):
"""Raises error on invalid distance matrix file."""
# Asymmetric.
with self.assertRaises(DistanceMatrixError):
_ = DistanceMatrix.from_file(self.dm_2x2_asym_f)
def test_from_file_invalid_input(self):
"""Raises error on invalid distance matrix file."""
# Asymmetric.
with self.assertRaises(DistanceMatrixError):
DistanceMatrix.from_file(self.dm_2x2_asym_f)
def setup(self):
with open(get_data_path('PCoA_sample_data_3'), 'U') as lines:
dist_matrix = DistanceMatrix.from_file(lines)
self.ordination = PCoA(dist_matrix)
self.ids = ['PC.636', 'PC.635', 'PC.356', 'PC.481', 'PC.354', 'PC.593',
'PC.355', 'PC.607', 'PC.634']
def compare_categories(dm_fp, map_fp, method, categories, num_perms, out_dir):
"""Runs the specified statistical method using the category of interest.
This method does not return anything; all output is written to results
files in out_dir.
Arguments:
dm_fp - filepath to the input distance matrix
map_fp - filepath to the input metadata mapping file
categories - list of categories in the metadata mapping file to
consider in the statistical test. Multiple categories will only be
considered if method is 'best', otherwise only the first category
will be considered
num_perms - the number of permutations to use when calculating the
p-value. If method is 'best' or 'morans_i', this parameter will be
ignored as they are not permutation-based methods
out_dir - path to the output directory where results files will be
written. It is assumed that this directory already exists and we
have write permissions to it
"""
# Make sure we were passed a list of categories, not a single string.
if not isinstance(categories, ListType):
raise TypeError("The supplied categories must be a list of "
"strings.")
# Special case: we do not allow SampleID as it is not a category, neither
# in data structure representation nor in terms of a statistical test (no
# groups are formed since all entries are unique IDs).
if 'SampleID' in categories:
raise ValueError("Cannot use SampleID as a category because it is a "
"unique identifier for each sample, and thus does "
"not create groups of samples (nor can it be used as "
"a numeric category in Moran's I or BEST analyses). "
"Please use a different metadata column to perform "
"statistical tests on.")
# Parse the mapping file and distance matrix.
with open(map_fp, 'U') as map_f:
md_map = MetadataMap.parseMetadataMap(map_f)
with open(dm_fp, 'U') as dm_f:
dm = DistanceMatrix.from_file(dm_f)
# Remove any samples from the mapping file that aren't in the distance
# matrix (important for validation checks). Use strict=True so that an
# error is raised if the distance matrix contains any samples that aren't
# in the mapping file.
md_map.filterSamples(dm.ids, strict=True)
# Run the specified statistical method.
if method in ['adonis', 'morans_i', 'mrpp', 'permdisp', 'dbrda']:
# These methods are run in R. Input validation must be done here before
# running the R commands. The pure-Python implementations perform all
# validation in the classes in the stats module.
# Check to make sure all categories passed in are in mapping file and
# are not all the same value.
for category in categories:
if not category in md_map.CategoryNames:
raise ValueError("Category '%s' not found in mapping file "
"columns." % category)
if md_map.hasSingleCategoryValue(category):
raise ValueError("All values in category '%s' are the "
"same. The statistical method '%s' cannot "
"operate on a category that creates only "
"a single group of samples (e.g. there "
"are no 'between' distances because "
"there is only a single group)." %
(category, method))
# Build the command arguments string.
command_args = [
'-d %s -m %s -c %s -o %s' % (dm_fp, map_fp, categories[0], out_dir)
]
if method == 'morans_i':
# Moran's I requires only numeric categories.
for category in categories:
if not md_map.isNumericCategory(category):
raise TypeError(
"The category '%s' is not numeric. Not "
"all values could be converted to numbers." % category)
else:
# The rest require groups of samples, so the category values cannot
# all be unique.
for category in categories:
if md_map.hasUniqueCategoryValues(category):
raise ValueError("All values in category '%s' are unique. "
"This statistical method cannot operate "
"on a category with unique values (e.g. "
"there are no 'within' distances because "
"each group of samples contains only a "
"single sample)." % category)
# Only Moran's I doesn't accept a number of permutations.
if num_perms < 0:
raise ValueError("The number of permutations must be greater "
"than or equal to zero.")
command_args[0] += ' -n %d' % num_perms
rex = RExecutor(TmpDir=get_qiime_temp_dir())
rex(command_args, '%s.r' % method, output_dir=out_dir)
elif method == 'anosim':
anosim = Anosim(md_map, dm, categories[0])
anosim_results = anosim(num_perms)
out_f = open(join(out_dir, '%s_results.txt' % method), 'w+')
out_f.write(format_anosim_results(anosim_results))
out_f.close()
elif method == 'best':
best = Best(dm, md_map, categories)
best_results = best()
out_f = open(join(out_dir, '%s_results.txt' % method), 'w+')
out_f.write(format_best_results(best_results))
out_f.close()
elif method == 'permanova':
permanova = Permanova(md_map, dm, categories[0])
permanova_results = permanova(num_perms)
out_f = open(join(out_dir, '%s_results.txt' % method), 'w+')
out_f.write(format_permanova_results(permanova_results))
out_f.close()
else:
raise ValueError("Unrecognized method '%s'. Valid methods: %r" %
(method, methods))
def compare_categories(dm_fp, map_fp, method, categories, num_perms, out_dir):
"""Runs the specified statistical method using the category of interest.
This method does not return anything; all output is written to results
files in out_dir.
Arguments:
dm_fp - filepath to the input distance matrix
map_fp - filepath to the input metadata mapping file
categories - list of categories in the metadata mapping file to
consider in the statistical test. Multiple categories will only be
considered if method is 'best', otherwise only the first category
will be considered
num_perms - the number of permutations to use when calculating the
p-value. If method is 'best' or 'morans_i', this parameter will be
ignored as they are not permutation-based methods
out_dir - path to the output directory where results files will be
written. It is assumed that this directory already exists and we
have write permissions to it
"""
# Make sure we were passed a list of categories, not a single string.
if not isinstance(categories, ListType):
raise TypeError("The supplied categories must be a list of "
"strings.")
# Special case: we do not allow SampleID as it is not a category, neither
# in data structure representation nor in terms of a statistical test (no
# groups are formed since all entries are unique IDs).
if 'SampleID' in categories:
raise ValueError("Cannot use SampleID as a category because it is a "
"unique identifier for each sample, and thus does "
"not create groups of samples (nor can it be used as "
"a numeric category in Moran's I or BEST analyses). "
"Please use a different metadata column to perform "
"statistical tests on.")
with open(dm_fp, 'U') as dm_f:
dm = DistanceMatrix.from_file(dm_f)
# These methods are in skbio. There are still methods in qiime.stats that
# need to be ported to skbio, at which point a lot of this logic can be
# simplified.
if method in ('anosim', 'permanova'):
if method == 'anosim':
method_cls = ANOSIM
elif method == 'permanova':
method_cls = PERMANOVA
else:
# Should never get here...
pass
with open(map_fp, 'U') as map_f:
md_dict = parse_mapping_file_to_dict(map_f)[0]
df = pd.DataFrame.from_dict(md_dict, orient='index')
method_inst = method_cls(dm, df, column=categories[0])
results = method_inst(num_perms)
with open(join(out_dir, '%s_results.txt' % method), 'w') as out_f:
out_f.write(results.summary())
else:
# Remove any samples from the mapping file that aren't in the distance
# matrix (important for validation checks). Use strict=True so that an
# error is raised if the distance matrix contains any samples that
# aren't in the mapping file.
with open(map_fp, 'U') as map_f:
md_map = MetadataMap.parseMetadataMap(map_f)
md_map.filterSamples(dm.ids, strict=True)
# Run the specified statistical method.
if method in ['adonis', 'morans_i', 'mrpp', 'permdisp', 'dbrda']:
# These methods are run in R. Input validation must be done here
# before running the R commands. The pure-Python implementations
# perform all validation in the classes in the stats module.
# Check to make sure all categories passed in are in mapping file
# and are not all the same value.
for category in categories:
if not category in md_map.CategoryNames:
raise ValueError("Category '%s' not found in mapping file "
"columns." % category)
if md_map.hasSingleCategoryValue(category):
raise ValueError("All values in category '%s' are the "
"same. The statistical method '%s' "
"cannot operate on a category that "
"creates only a single group of samples "
"(e.g. there are no 'between' distances "
"because there is only a single group)."
% (category, method))
# Build the command arguments string.
command_args = ['-d %s -m %s -c %s -o %s'
% (dm_fp, map_fp, categories[0], out_dir)]
if method == 'morans_i':
# Moran's I requires only numeric categories.
for category in categories:
if not md_map.isNumericCategory(category):
raise TypeError("The category '%s' is not numeric. "
"Not all values could be converted to "
"numbers." % category)
else:
# The rest require groups of samples, so the category values
# cannot all be unique.
for category in categories:
if md_map.hasUniqueCategoryValues(category):
raise ValueError("All values in category '%s' are "
"unique. This statistical method "
"cannot operate on a category with "
"unique values (e.g. there are no "
"'within' distances because each "
"group of samples contains only a "
"single sample)." % category)
# Only Moran's I doesn't accept a number of permutations.
if num_perms < 0:
raise ValueError("The number of permutations must be "
"greater than or equal to zero.")
command_args[0] += ' -n %d' % num_perms
rex = RExecutor(TmpDir=get_qiime_temp_dir())
rex(command_args, '%s.r' % method, output_dir=out_dir)
elif method == 'best':
best = Best(dm, md_map, categories)
best_results = best()
with open(join(out_dir, '%s_results.txt' % method), 'w') as out_f:
out_f.write(format_best_results(best_results))
else:
raise ValueError("Unrecognized method '%s'. Valid methods: %r"
% (method, methods))
def setUp(self):
# The test dataset used here is a subset of the Lauber et al. 2009
# "88 Soils" dataset. It has been altered to exercise various aspects
# of the code, including (but not limited to):
#
# - order of distance matrix IDs and IDs in data frame (metadata) are
# not exactly the same
# - data frame has an extra sample that is not in the distance matrix
# - this extra sample has non-numeric and missing values in some of its
# cells
#
# Additional variations of the distance matrix and data frame are used
# to test different orderings of rows/columns, extra non-numeric data
# frame columns, etc.
#
# This dataset is also useful because it is non-trivial in size (6
# samples, 11 environment variables) and it includes positive/negative
# floats and integers in the data frame.
self.dm = DistanceMatrix.from_file(get_data_path('dm.txt'))
# Reordered rows and columns (i.e., different ID order). Still
# conceptually the same distance matrix.
self.dm_reordered = DistanceMatrix.from_file(
get_data_path('dm_reordered.txt'))
self.df = pd.read_csv(get_data_path('df.txt'), sep='\t', index_col=0)
# Similar to the above data frame, except that it has an extra
# non-numeric column, and some of the other rows and columns have been
# reordered.
self.df_extra_column = pd.read_csv(
get_data_path('df_extra_column.txt'), sep='\t', index_col=0)
# All columns in the original data frame (these are all numeric
# columns).
self.cols = self.df.columns.tolist()
# This second dataset is derived from vegan::bioenv's example dataset
# (varespec and varechem). The original dataset includes a site x
# species table (e.g., OTU table) and a data frame of environmental
# variables. Since the bioenv function defined here accepts a distance
# matrix, we use a Bray-Curtis distance matrix that is derived from the
# site x species table (this matches what is done by vegan::bioenv when
# provided an OTU table, using their default distance measure). The
# data frame only includes the numeric environmental variables we're
# interested in for these tests: log(N), P, K, Ca, pH, Al
self.dm_vegan = DistanceMatrix.from_file(
get_data_path('bioenv_dm_vegan.txt'))
self.df_vegan = pd.read_csv(
get_data_path('bioenv_df_vegan.txt'), sep='\t',
converters={0: str})
self.df_vegan.set_index('#SampleID', inplace=True)
# Load expected results.
self.exp_results = pd.read_csv(get_data_path('exp_results.txt'),
sep='\t', index_col=0)
self.exp_results_single_column = pd.read_csv(
get_data_path('exp_results_single_column.txt'), sep='\t',
index_col=0)
self.exp_results_different_column_order = pd.read_csv(
get_data_path('exp_results_different_column_order.txt'), sep='\t',
index_col=0)
self.exp_results_vegan = pd.read_csv(
get_data_path('bioenv_exp_results_vegan.txt'), sep='\t',
index_col=0)
def setUp(self):
# The test dataset used here is a subset of the Lauber et al. 2009
# "88 Soils" dataset. It has been altered to exercise various aspects
# of the code, including (but not limited to):
#
# - order of distance matrix IDs and IDs in data frame (metadata) are
# not exactly the same
# - data frame has an extra sample that is not in the distance matrix
# - this extra sample has non-numeric and missing values in some of its
# cells
#
# Additional variations of the distance matrix and data frame are used
# to test different orderings of rows/columns, extra non-numeric data
# frame columns, etc.
#
# This dataset is also useful because it is non-trivial in size (6
# samples, 11 environment variables) and it includes positive/negative
# floats and integers in the data frame.
self.dm = DistanceMatrix.from_file(get_data_path('dm.txt'))
# Reordered rows and columns (i.e., different ID order). Still
# conceptually the same distance matrix.
self.dm_reordered = DistanceMatrix.from_file(
get_data_path('dm_reordered.txt'))
self.df = pd.read_csv(get_data_path('df.txt'), sep='\t', index_col=0)
# Similar to the above data frame, except that it has an extra
# non-numeric column, and some of the other rows and columns have been
# reordered.
self.df_extra_column = pd.read_csv(
get_data_path('df_extra_column.txt'), sep='\t', index_col=0)
# All columns in the original data frame (these are all numeric
# columns).
self.cols = self.df.columns.tolist()
# This second dataset is derived from vegan::bioenv's example dataset
# (varespec and varechem). The original dataset includes a site x
# species table (e.g., OTU table) and a data frame of environmental
# variables. Since the bioenv function defined here accepts a distance
# matrix, we use a Bray-Curtis distance matrix that is derived from the
# site x species table (this matches what is done by vegan::bioenv when
# provided an OTU table, using their default distance measure). The
# data frame only includes the numeric environmental variables we're
# interested in for these tests: log(N), P, K, Ca, pH, Al
self.dm_vegan = DistanceMatrix.from_file(
get_data_path('bioenv_dm_vegan.txt'))
self.df_vegan = pd.read_csv(get_data_path('bioenv_df_vegan.txt'),
sep='\t',
converters={0: str})
self.df_vegan.set_index('#SampleID', inplace=True)
# Load expected results.
self.exp_results = pd.read_csv(get_data_path('exp_results.txt'),
sep='\t',
index_col=0)
self.exp_results_single_column = pd.read_csv(
get_data_path('exp_results_single_column.txt'),
sep='\t',
index_col=0)
self.exp_results_different_column_order = pd.read_csv(
get_data_path('exp_results_different_column_order.txt'),
sep='\t',
index_col=0)
self.exp_results_vegan = pd.read_csv(
get_data_path('bioenv_exp_results_vegan.txt'),
sep='\t',
index_col=0)
