def _setup_multiple_weighted_unifrac(counts, otu_ids, tree, normalized,
validate):
""" Create optimized pdist-compatible weighted UniFrac function
Parameters
----------
counts : 2D array_like of ints or floats
Matrix containing count/abundance data where each row contains counts
of observations in a given sample.
otu_ids: list, np.array
Vector of OTU ids corresponding to tip names in ``tree``. Must be the
same length as ``u_counts`` and ``v_counts``. These IDs do not need to
be in tip order with respect to the tree.
tree: skbio.TreeNode
Tree relating the OTUs in otu_ids. The set of tip names in the tree can
be a superset of ``otu_ids``, but not a subset.
validate: bool, optional
If `False`, validation of the input won't be performed.
Returns
-------
function
Optimized pairwise unweighted UniFrac calculator that can be passed
to ``scipy.spatial.distance.pdist``.
2D np.array of ints, floats
Counts of all nodes in ``tree``.
"""
counts_by_node, tree_index, branch_lengths = \
_setup_multiple_unifrac(counts, otu_ids, tree, validate)
tip_indices = _get_tip_indices(tree_index)
if normalized:
node_to_root_distances = _tip_distances(branch_lengths, tree,
tip_indices)
def f(u_node_counts, v_node_counts):
u_total_count = np.take(u_node_counts, tip_indices).sum()
v_total_count = np.take(v_node_counts, tip_indices).sum()
u = _weighted_unifrac_normalized(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths,
node_to_root_distances)
return u
else:
def f(u_node_counts, v_node_counts):
u_total_count = np.take(u_node_counts, tip_indices).sum()
v_total_count = np.take(v_node_counts, tip_indices).sum()
u, _, _ = _weighted_unifrac(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths)
return u
return f, counts_by_node
def _setup_multiple_weighted_unifrac(counts, otu_ids, tree, normalized,
validate):
""" Create optimized pdist-compatible weighted UniFrac function
Parameters
----------
counts : 2D array_like of ints or floats
Matrix containing count/abundance data where each row contains counts
of observations in a given sample.
otu_ids: list, np.array
Vector of OTU ids corresponding to tip names in ``tree``. Must be the
same length as ``u_counts`` and ``v_counts``. These IDs do not need to
be in tip order with respect to the tree.
tree: skbio.TreeNode
Tree relating the OTUs in otu_ids. The set of tip names in the tree can
be a superset of ``otu_ids``, but not a subset.
validate: bool, optional
If `False`, validation of the input won't be performed.
Returns
-------
function
Optimized pairwise unweighted UniFrac calculator that can be passed
to ``scipy.spatial.distance.pdist``.
2D np.array of ints, floats
Counts of all nodes in ``tree``.
"""
counts_by_node, tree_index, branch_lengths = \
_setup_multiple_unifrac(counts, otu_ids, tree, validate)
tip_indices = _get_tip_indices(tree_index)
if normalized:
node_to_root_distances = _tip_distances(branch_lengths, tree,
tip_indices)
def f(u_node_counts, v_node_counts):
u_total_count = np.take(u_node_counts, tip_indices).sum()
v_total_count = np.take(v_node_counts, tip_indices).sum()
u = _weighted_unifrac_normalized(
u_node_counts, v_node_counts, u_total_count, v_total_count,
branch_lengths, node_to_root_distances)
return u
else:
def f(u_node_counts, v_node_counts):
u_total_count = np.take(u_node_counts, tip_indices).sum()
v_total_count = np.take(v_node_counts, tip_indices).sum()
u, _, _ = _weighted_unifrac(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths)
return u
return f, counts_by_node
def weighted_unifrac(u_counts,
v_counts,
otu_ids,
tree,
normalized=_normalize_weighted_unifrac_by_default,
validate=True):
""" Compute weighted UniFrac with or without branch length normalization
Parameters
----------
u_counts, v_counts: list, np.array
Vectors of counts/abundances of OTUs for two samples. Must be equal
length.
otu_ids: list, np.array
Vector of OTU ids corresponding to tip names in ``tree``. Must be the
same length as ``u_counts`` and ``v_counts``.
tree: skbio.TreeNode
Tree relating the OTUs in otu_ids. The set of tip names in the tree can
be a superset of ``otu_ids``, but not a subset.
normalized: boolean, optional
If ``True``, apply branch length normalization, which is described in
[1]_. Resulting distances will then be in the range ``[0, 1]``.
validate: bool, optional
If `False`, validation of the input won't be performed. This step can
be slow, so if validation is run elsewhere it can be disabled here.
However, invalid input data can lead to invalid results or error
messages that are hard to interpret, so this step should not be
bypassed if you're not certain that your input data are valid. See
Notes for the description of what validation entails so you can
determine if you can safely disable validation.
Returns
-------
float
The weighted UniFrac distance between the two samples.
Raises
------
ValueError, MissingNodeError, DuplicateNodeError
If validation fails (see description of validation in Notes). Exact
error will depend on what was invalid.
See Also
--------
unweighted_unifrac
skbio.diversity.beta_diversity
Notes
-----
Weighted UniFrac was originally described in [1]_, which includes a
discussion of unweighted (qualitative) versus weighted (quantitiative)
diversity metrics. Deeper mathemtical discussions of this metric is
presented in [2]_.
If computing weighted UniFrac for multiple pairs of samples, using
``skbio.diversity.beta_diversity`` will be much faster than calling this
function individually on each sample.
This implementation differs from that in PyCogent (and therefore QIIME
versions less than 2.0.0) by imposing a few additional restrictions on the
inputs. First, the input tree must be rooted. In PyCogent, if an unrooted
tree was provided that had a single trifurcating node (a newick convention
for unrooted trees) that node was considered the root of the tree. Next,
all OTU IDs must be tips in the tree. PyCogent would silently ignore OTU
IDs that were not present the tree. To reproduce UniFrac results from
PyCogent with scikit-bio, ensure that your PyCogent UniFrac calculations
are performed on a rooted tree and that all OTU IDs are present in the
tree.
This implementation of weighted UniFrac is the array-based implementation
described in [3]_.
Validation of input data confirms the following:
* ``counts`` data can be safely cast to integers
* there are no negative values in ``counts``
* ``counts`` has the correct number of dimensions
* all vectors in ``counts`` are of equal length
* ``otu_ids`` does not contain duplicate values
* the length of each ``counts`` vector is equal to ``len(otu_ids)``
* ``tree`` is rooted
* ``tree`` has more than one node
* all nodes in ``tree`` except for the root node have branch lengths
* all tip names in ``tree`` are unique
* all ``otu_ids`` correspond to tip names in ``tree``
References
----------
.. [1] Lozupone, C. A., Hamady, M., Kelley, S. T. & Knight, R. Quantitative
and qualitative beta diversity measures lead to different insights into
factors that structure microbial communities. Appl. Environ. Microbiol.
73, 1576-1585 (2007).
.. [2] Lozupone, C., Lladser, M. E., Knights, D., Stombaugh, J. & Knight,
R. UniFrac: an effective distance metric for microbial community
comparison. ISME J. 5, 169-172 (2011).
.. [3] Hamady M, Lozupone C, Knight R. Fast UniFrac: facilitating high-
throughput phylogenetic analyses of microbial communities including
analysis of pyrosequencing and PhyloChip data. ISME J. 4(1):17-27
(2010).
Examples
--------
Assume we have the following abundance data for two samples, ``u`` and
``v``, represented as a pair of counts vectors. These counts represent the
number of times specific Operational Taxonomic Units, or OTUs, were
observed in each of the samples.
>>> u_counts = [1, 0, 0, 4, 1, 2, 3, 0]
>>> v_counts = [0, 1, 1, 6, 0, 1, 0, 0]
Because UniFrac is a phylogenetic diversity metric, we need to know which
OTU each count corresponds to, which we'll provide as ``otu_ids``.
>>> otu_ids = ['OTU1', 'OTU2', 'OTU3', 'OTU4', 'OTU5', 'OTU6', 'OTU7',
... 'OTU8']
We also need a phylogenetic tree that relates the OTUs to one another.
>>> from io import StringIO
>>> from skbio import TreeNode
>>> tree = TreeNode.read(StringIO(
... u'(((((OTU1:0.5,OTU2:0.5):0.5,OTU3:1.0):1.0):0.0,'
... u'(OTU4:0.75,(OTU5:0.5,((OTU6:0.33,OTU7:0.62):0.5'
... u',OTU8:0.5):0.5):0.5):1.25):0.0)root;'))
Compute the weighted UniFrac distance between the samples.
>>> from skbio.diversity.beta import weighted_unifrac
>>> wu = weighted_unifrac(u_counts, v_counts, otu_ids, tree)
>>> print(round(wu, 2))
1.54
Compute the weighted UniFrac distance between the samples including
branch length normalization so the value falls in the range ``[0.0, 1.0]``.
>>> wu = weighted_unifrac(u_counts, v_counts, otu_ids, tree,
... normalized=True)
>>> print(round(wu, 2))
0.33
"""
u_node_counts, v_node_counts, u_total_count, v_total_count, tree_index =\
_setup_pairwise_unifrac(u_counts, v_counts, otu_ids, tree, validate,
normalized=normalized, unweighted=False)
branch_lengths = tree_index['length']
if normalized:
tip_indices = _get_tip_indices(tree_index)
node_to_root_distances = _tip_distances(branch_lengths, tree,
tip_indices)
return _weighted_unifrac_normalized(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths,
node_to_root_distances)
else:
return _weighted_unifrac(u_node_counts, v_node_counts, u_total_count,
v_total_count, branch_lengths)[0]
def weighted_unifrac(u_counts, v_counts, otu_ids, tree,
normalized=_normalize_weighted_unifrac_by_default,
validate=True):
""" Compute weighted UniFrac with or without branch length normalization
Parameters
----------
u_counts, v_counts: list, np.array
Vectors of counts/abundances of OTUs for two samples. Must be equal
length.
otu_ids: list, np.array
Vector of OTU ids corresponding to tip names in ``tree``. Must be the
same length as ``u_counts`` and ``v_counts``.
tree: skbio.TreeNode
Tree relating the OTUs in otu_ids. The set of tip names in the tree can
be a superset of ``otu_ids``, but not a subset.
normalized: boolean, optional
If ``True``, apply branch length normalization, which is described in
[1]_. Resulting distances will then be in the range ``[0, 1]``.
validate: bool, optional
If `False`, validation of the input won't be performed. This step can
be slow, so if validation is run elsewhere it can be disabled here.
However, invalid input data can lead to invalid results or error
messages that are hard to interpret, so this step should not be
bypassed if you're not certain that your input data are valid. See
:mod:`skbio.diversity` for the description of what validation entails
so you can determine if you can safely disable validation.
Returns
-------
float
The weighted UniFrac distance between the two samples.
Raises
------
ValueError, MissingNodeError, DuplicateNodeError
If validation fails. Exact error will depend on what was invalid.
See Also
--------
unweighted_unifrac
skbio.diversity
skbio.diversity.beta_diversity
Notes
-----
Weighted UniFrac was originally described in [1]_, which includes a
discussion of unweighted (qualitative) versus weighted (quantitiative)
diversity metrics. Deeper mathemtical discussions of this metric is
presented in [2]_.
If computing weighted UniFrac for multiple pairs of samples, using
``skbio.diversity.beta_diversity`` will be much faster than calling this
function individually on each sample.
This implementation differs from that in PyCogent (and therefore QIIME
versions less than 2.0.0) by imposing a few additional restrictions on the
inputs. First, the input tree must be rooted. In PyCogent, if an unrooted
tree was provided that had a single trifurcating node (a newick convention
for unrooted trees) that node was considered the root of the tree. Next,
all OTU IDs must be tips in the tree. PyCogent would silently ignore OTU
IDs that were not present the tree. To reproduce UniFrac results from
PyCogent with scikit-bio, ensure that your PyCogent UniFrac calculations
are performed on a rooted tree and that all OTU IDs are present in the
tree.
This implementation of weighted UniFrac is the array-based implementation
described in [3]_.
References
----------
.. [1] Lozupone, C. A., Hamady, M., Kelley, S. T. & Knight, R. Quantitative
and qualitative beta diversity measures lead to different insights into
factors that structure microbial communities. Appl. Environ. Microbiol.
73, 1576-1585 (2007).
.. [2] Lozupone, C., Lladser, M. E., Knights, D., Stombaugh, J. & Knight,
R. UniFrac: an effective distance metric for microbial community
comparison. ISME J. 5, 169-172 (2011).
.. [3] Hamady M, Lozupone C, Knight R. Fast UniFrac: facilitating high-
throughput phylogenetic analyses of microbial communities including
analysis of pyrosequencing and PhyloChip data. ISME J. 4(1):17-27
(2010).
Examples
--------
Assume we have the following abundance data for two samples, ``u`` and
``v``, represented as a pair of counts vectors. These counts represent the
number of times specific Operational Taxonomic Units, or OTUs, were
observed in each of the samples.
>>> u_counts = [1, 0, 0, 4, 1, 2, 3, 0]
>>> v_counts = [0, 1, 1, 6, 0, 1, 0, 0]
Because UniFrac is a phylogenetic diversity metric, we need to know which
OTU each count corresponds to, which we'll provide as ``otu_ids``.
>>> otu_ids = ['OTU1', 'OTU2', 'OTU3', 'OTU4', 'OTU5', 'OTU6', 'OTU7',
... 'OTU8']
We also need a phylogenetic tree that relates the OTUs to one another.
>>> from io import StringIO
>>> from skbio import TreeNode
>>> tree = TreeNode.read(StringIO(
... '(((((OTU1:0.5,OTU2:0.5):0.5,OTU3:1.0):1.0):0.0,'
... '(OTU4:0.75,(OTU5:0.5,((OTU6:0.33,OTU7:0.62):0.5'
... ',OTU8:0.5):0.5):0.5):1.25):0.0)root;'))
Compute the weighted UniFrac distance between the samples.
>>> from skbio.diversity.beta import weighted_unifrac
>>> wu = weighted_unifrac(u_counts, v_counts, otu_ids, tree)
>>> print(round(wu, 2))
1.54
Compute the weighted UniFrac distance between the samples including
branch length normalization so the value falls in the range ``[0.0, 1.0]``.
>>> wu = weighted_unifrac(u_counts, v_counts, otu_ids, tree,
... normalized=True)
>>> print(round(wu, 2))
0.33
"""
u_node_counts, v_node_counts, u_total_count, v_total_count, tree_index =\
_setup_pairwise_unifrac(u_counts, v_counts, otu_ids, tree, validate,
normalized=normalized, unweighted=False)
branch_lengths = tree_index['length']
if normalized:
tip_indices = _get_tip_indices(tree_index)
node_to_root_distances = _tip_distances(branch_lengths, tree,
tip_indices)
return _weighted_unifrac_normalized(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths,
node_to_root_distances)
else:
return _weighted_unifrac(u_node_counts, v_node_counts,
u_total_count, v_total_count,
branch_lengths)[0]