def ilr_phylogenetic(
table: pd.DataFrame,
tree: skbio.TreeNode,
pseudocount: float = 0.5) -> (pd.DataFrame, skbio.TreeNode):
t = tree.copy()
t.bifurcate()
t = rename_internal_nodes(t)
return ilr_transform(add_pseudocount(table, pseudocount), t), t
def test_add_pseudocount_3(self):
table = pd.DataFrame([[1.7, 1.3, 0.5], [1.5, 3.2, 1.1]],
columns=['a', 'b', 'c'],
index=[1, 2])
obs = add_pseudocount(table, 2.0)
exp = pd.DataFrame([[1.7, 1.3, 0.5], [1.5, 3.2, 1.1]],
columns=['a', 'b', 'c'],
index=[1, 2])
pdt.assert_frame_equal(obs, exp)
def test_add_pseudocount_2(self):
table = pd.DataFrame(
[[1.0, 2.0, 2.5], [0.5, 0.0, 1.0], [0.0, 0.5, 1.0]],
columns=['a', 'b', 'c'], index=[1, 2, 3])
obs = add_pseudocount(table, 0.5)
exp = pd.DataFrame(
[[1.0, 2.0, 2.5], [0.5, 0.5, 1.0], [0.5, 0.5, 1.0]],
columns=['a', 'b', 'c'], index=[1, 2, 3])
pdt.assert_frame_equal(obs, exp)
def test_add_pseudocount(self):
table = pd.DataFrame(
[[1., 2., 3., 4.], [1., 0., 2., 1.], [0., 2., 1., 3.]],
columns=['a', 'b', 'c', 'd'], index=[1, 2, 3])
obs = add_pseudocount(table)
exp = pd.DataFrame(
[[1., 2., 3., 4.], [1., 0.5, 2., 1.], [0.5, 2., 1., 3.]],
columns=['a', 'b', 'c', 'd'], index=[1, 2, 3])
pdt.assert_frame_equal(obs, exp)
def correlation_clustering(table: pd.DataFrame, pseudocount: float = 0.5
) -> skbio.TreeNode:
""" Builds a tree for features based on correlation.
Parameters
----------
table : pd.DataFrame
Contingency table where rows are samples and columns are features.
In addition, the table must have strictly nonzero values.
Returns
-------
skbio.TreeNode
Represents the partitioning of features with respect to correlation.
"""
t = correlation_linkage(add_pseudocount(table, pseudocount))
return t
def ilr_phylogenetic_ordination(
table: pd.DataFrame,
tree: skbio.TreeNode,
pseudocount: float = 0.5,
top_k_var: int = 10,
clades: list = None
) -> (OrdinationResults, skbio.TreeNode, pd.DataFrame):
t = tree.copy()
t.bifurcate()
_table, _tree = match_tips(table, t)
_tree = rename_internal_nodes(_tree)
if not clades:
in_nodes = [n.name for n in _tree.levelorder() if not n.is_tip()]
basis = _balance_basis(_tree)[0]
_table = add_pseudocount(_table, pseudocount)
basis = pd.DataFrame(basis.T, index=_table.columns, columns=in_nodes)
balances = np.log(_table) @ basis
var = balances.var(axis=0).sort_values(ascending=False)
clades = var.index[:top_k_var]
balances = balances[clades]
basis = basis[clades]
else:
clades = clades[0].split(',')
balances, basis = _fast_ilr(_tree, _table, clades, pseudocount=0.5)
var = balances.var(axis=0).sort_values(ascending=False)
balances.index.name = 'sampleid'
# feature metadata
eigvals = var
prop = var[clades] / var.sum()
balances = OrdinationResults(
short_method_name='ILR',
long_method_name='Phylogenetic Isometric Log Ratio Transform',
samples=balances,
features=pd.DataFrame(np.eye(len(clades)), index=clades),
eigvals=eigvals,
proportion_explained=prop)
basis.index.name = 'featureid'
return balances, _tree, basis
def dendrogram_heatmap(output_dir: str, table: pd.DataFrame,
tree: TreeNode,
metadata: qiime2.CategoricalMetadataColumn,
pseudocount: float = 0.5,
ndim: int = 10, method: str = 'clr',
color_map: str = 'viridis'):
table, tree = match_tips(add_pseudocount(table, pseudocount), tree)
nodes = [n.name for n in tree.levelorder() if not n.is_tip()]
nlen = min(ndim, len(nodes))
numerator_color, denominator_color = '#fb9a99', '#e31a1c'
highlights = pd.DataFrame([[numerator_color, denominator_color]] * nlen,
index=nodes[:nlen])
if method == 'clr':
mat = pd.DataFrame(clr(centralize(table)),
index=table.index,
columns=table.columns)
elif method == 'log':
mat = pd.DataFrame(np.log(table),
index=table.index,
columns=table.columns)
c = metadata.to_series()
table, c = match(table, c)
# TODO: There are a few hard-coded constants here
# will need to have some adaptive defaults set in the future
fig = heatmap(mat, tree, c, highlights, cmap=color_map,
highlight_width=0.01, figsize=(12, 8))
fig.savefig(os.path.join(output_dir, 'heatmap.svg'))
fig.savefig(os.path.join(output_dir, 'heatmap.pdf'))
css = r"""
.square {
float: left;
width: 100px;
height: 20px;
margin: 5px;
border: 1px solid rgba(0, 0, 0, .2);
}
.numerator {
background: %s;
}
.denominator {
background: %s;
}
""" % (numerator_color, denominator_color)
index_fp = os.path.join(output_dir, 'index.html')
with open(index_fp, 'w') as index_f:
index_f.write('<html><body>\n')
index_f.write('<h1>Dendrogram heatmap</h1>\n')
index_f.write('<img src="heatmap.svg" alt="heatmap">')
index_f.write('<a href="heatmap.pdf">')
index_f.write('Download as PDF</a><br>\n')
index_f.write('<style>%s</style>' % css)
index_f.write('<div class="square numerator">'
'Numerator<br/></div>')
index_f.write('<div class="square denominator">'
'Denominator<br/></div>')
index_f.write('</body></html>\n')
def balance_taxonomy(output_dir: str, table: pd.DataFrame, tree: TreeNode,
taxonomy: pd.DataFrame,
balance_name: str,
pseudocount: float = 0.5,
taxa_level: int = 0,
n_features: int = 10,
threshold: float = None,
metadata: qiime2.MetadataColumn = None) -> None:
if threshold is not None and isinstance(metadata,
qiime2.CategoricalMetadataColumn):
raise ValueError('Categorical metadata column detected. Only specify '
'a threshold when using a numerical metadata column.')
# make sure that the table and tree match up
table, tree = match_tips(add_pseudocount(table, pseudocount), tree)
# parse out headers for taxonomy
taxa_data = list(taxonomy['Taxon'].apply(lambda x: x.split(';')).values)
taxa_df = pd.DataFrame(taxa_data, index=taxonomy.index)
# fill in NAs
def f(x):
y = np.array(list(map(lambda k: k is not None, x)))
i = max(0, np.where(y)[0][-1])
x[np.logical_not(y)] = [x[i]] * np.sum(np.logical_not(y))
return x
taxa_df = taxa_df.apply(f, axis=1)
num_clade = tree.find(balance_name).children[NUMERATOR]
denom_clade = tree.find(balance_name).children[DENOMINATOR]
if num_clade.is_tip():
num_features = pd.DataFrame(
{num_clade.name: taxa_df.loc[num_clade.name]}
).T
r = 1
else:
num_features = taxa_df.loc[num_clade.subset()]
r = len(list(num_clade.tips()))
if denom_clade.is_tip():
denom_features = pd.DataFrame(
{denom_clade.name: taxa_df.loc[denom_clade.name]}
).T
s = 1
else:
denom_features = taxa_df.loc[denom_clade.subset()]
s = len(list(denom_clade.tips()))
b = (np.log(table.loc[:, num_features.index]).mean(axis=1) -
np.log(table.loc[:, denom_features.index]).mean(axis=1))
b = b * np.sqrt(r * s / (r + s))
balances = pd.DataFrame(b, index=table.index,
columns=[balance_name])
# the actual colors for the numerator and denominator
num_color = sns.color_palette("Paired")[0]
denom_color = sns.color_palette("Paired")[1]
fig, (ax_num, ax_denom) = plt.subplots(2)
balance_barplots(tree, balance_name, taxa_level, taxa_df,
denom_color=denom_color, num_color=num_color,
axes=(ax_num, ax_denom))
ax_num.set_title(
r'$%s_{numerator} \; taxa \; (%d \; taxa)$' % (
balance_name, len(num_features)))
ax_denom.set_title(
r'$%s_{denominator} \; taxa \; (%d \; taxa)$' % (
balance_name, len(denom_features)))
ax_denom.set_xlabel('Number of unique taxa')
plt.tight_layout()
fig.savefig(os.path.join(output_dir, 'barplots.svg'))
fig.savefig(os.path.join(output_dir, 'barplots.pdf'))
dcat = None
multiple_cats = False
if metadata is not None:
fig2, ax = plt.subplots()
c = metadata.to_series()
data, c = match(balances, c)
data[c.name] = c
y = data[balance_name]
# check if continuous
if isinstance(metadata, qiime2.NumericMetadataColumn):
ax.scatter(c.values, y)
ax.set_xlabel(c.name)
if threshold is None:
threshold = c.mean()
dcat = c.apply(
lambda x: '%s < %f' % (c.name, threshold)
if x < threshold
else '%s > %f' % (c.name, threshold)
)
sample_palette = pd.Series(sns.color_palette("Set2", 2),
index=dcat.value_counts().index)
elif isinstance(metadata, qiime2.CategoricalMetadataColumn):
sample_palette = pd.Series(
sns.color_palette("Set2", len(c.value_counts())),
index=c.value_counts().index)
try:
pd.to_numeric(metadata.to_series())
except ValueError:
pass
else:
raise ValueError('Categorical metadata column '
f'{metadata.name!r} contains only numerical '
'values. At least one value must be '
'non-numerical.')
balance_boxplot(balance_name, data, y=c.name, ax=ax,
palette=sample_palette)
if len(c.value_counts()) > 2:
warnings.warn(
'More than 2 categories detected in categorical metadata '
'column. Proportion plots will not be displayed',
stacklevel=2)
multiple_cats = True
else:
dcat = c
else:
# Some other type of MetadataColumn
raise NotImplementedError()
ylabel = (r"$%s = \ln \frac{%s_{numerator}}"
"{%s_{denominator}}$") % (balance_name,
balance_name,
balance_name)
ax.set_title(ylabel, rotation=0)
ax.set_ylabel('log ratio')
fig2.savefig(os.path.join(output_dir, 'balance_metadata.svg'))
fig2.savefig(os.path.join(output_dir, 'balance_metadata.pdf'))
if not multiple_cats:
# Proportion plots
# first sort by clr values and calculate average fold change
ctable = pd.DataFrame(clr(centralize(table)),
index=table.index, columns=table.columns)
left_group = dcat.value_counts().index[0]
right_group = dcat.value_counts().index[1]
lidx, ridx = (dcat == left_group), (dcat == right_group)
if b.loc[lidx].mean() > b.loc[ridx].mean():
# double check ordering and switch if necessary
# careful - the left group is also commonly associated with
# the denominator.
left_group = dcat.value_counts().index[1]
right_group = dcat.value_counts().index[0]
lidx, ridx = (dcat == left_group), (dcat == right_group)
# we are not performing a statistical test here
# we're just trying to figure out a way to sort the data.
num_fold_change = ctable.loc[:, num_features.index].apply(
lambda x: ttest_ind(x[ridx], x[lidx])[0])
num_fold_change = num_fold_change.sort_values(
ascending=False
)
denom_fold_change = ctable.loc[:, denom_features.index].apply(
lambda x: ttest_ind(x[ridx], x[lidx])[0])
denom_fold_change = denom_fold_change.sort_values(
ascending=True
)
metadata = pd.DataFrame({dcat.name: dcat})
top_num_features = num_fold_change.index[:n_features]
top_denom_features = denom_fold_change.index[:n_features]
fig3, (ax_denom, ax_num) = plt.subplots(1, 2)
proportion_plot(
table, metadata,
category=metadata.columns[0],
left_group=left_group,
right_group=right_group,
feature_metadata=taxa_df,
label_col=taxa_level,
num_features=top_num_features,
denom_features=top_denom_features,
# Note that the syntax is funky and counter
# intuitive. This will need to be properly
# fixed here
# https://github.com/biocore/gneiss/issues/244
num_color=sample_palette.loc[right_group],
denom_color=sample_palette.loc[left_group],
axes=(ax_num, ax_denom))
# The below is overriding the default colors in the
# numerator / denominator this will also need to be fixed in
# https://github.com/biocore/gneiss/issues/244
max_ylim, min_ylim = ax_denom.get_ylim()
num_h, denom_h = n_features, n_features
space = (max_ylim - min_ylim) / (num_h + denom_h)
ymid = (max_ylim - min_ylim) * num_h
ymid = ymid / (num_h + denom_h) - 0.5 * space
ax_denom.axhspan(min_ylim, ymid,
facecolor=num_color,
zorder=0)
ax_denom.axhspan(ymid, max_ylim,
facecolor=denom_color,
zorder=0)
ax_num.axhspan(min_ylim, ymid,
facecolor=num_color,
zorder=0)
ax_num.axhspan(ymid, max_ylim,
facecolor=denom_color,
zorder=0)
fig3.subplots_adjust(
# the left side of the subplots of the figure
left=0.3,
# the right side of the subplots of the figure
right=0.9,
# the bottom of the subplots of the figure
bottom=0.1,
# the top of the subplots of the figure
top=0.9,
# the amount of width reserved for blank space
# between subplots
wspace=0,
# the amount of height reserved for white space
# between subplots
hspace=0.2,
)
fig3.savefig(os.path.join(output_dir, 'proportion_plot.svg'))
fig3.savefig(os.path.join(output_dir, 'proportion_plot.pdf'))
index_fp = os.path.join(output_dir, 'index.html')
with open(index_fp, 'w') as index_f:
index_f.write('<html><body>\n')
if metadata is not None:
index_f.write('<h1>Balance vs %s </h1>\n' % c.name)
index_f.write(('<img src="balance_metadata.svg" '
'alt="barplots">\n\n'
'<a href="balance_metadata.pdf">'
'Download as PDF</a><br>\n'))
if not multiple_cats:
index_f.write('<h1>Proportion Plot </h1>\n')
index_f.write(('<img src="proportion_plot.svg" '
'alt="proportions">\n\n'
'<a href="proportion_plot.pdf">'
'Download as PDF</a><br>\n'))
index_f.write(('<h1>Balance Taxonomy</h1>\n'
'<img src="barplots.svg" alt="barplots">\n\n'
'<a href="barplots.pdf">'
'Download as PDF</a><br>\n'
'<h3>Numerator taxa</h3>\n'
'<a href="numerator.csv">\n'
'Download as CSV</a><br>\n'
'<h3>Denominator taxa</h3>\n'
'<a href="denominator.csv">\n'
'Download as CSV</a><br>\n'))
num_features.to_csv(os.path.join(output_dir, 'numerator.csv'),
header=True, index=True)
denom_features.to_csv(os.path.join(output_dir, 'denominator.csv'),
header=True, index=True)
index_f.write('</body></html>\n')
def ilr_hierarchical(table: pd.DataFrame,
tree: skbio.TreeNode,
pseudocount: float = 0.5) -> pd.DataFrame:
return ilr_transform(add_pseudocount(table, pseudocount), tree)
