def multinomial(table: biom.Table,
metadata: Metadata,
formula: str,
training_column: str = None,
num_random_test_examples: int = 10,
epoch: int = 10,
batch_size: int = 5,
beta_prior: float = 1,
learning_rate: float = 0.1,
clipnorm: float = 10,
min_sample_count: int = 10,
min_feature_count: int = 10,
summary_interval: int = 60) -> (pd.DataFrame):
# load metadata and tables
metadata = metadata.to_dataframe()
# match them
table, metadata, design = match_and_filter(table, metadata, formula,
training_column,
num_random_test_examples,
min_sample_count,
min_feature_count)
# convert to dense representation
dense_table = table.to_dataframe().to_dense().T
# split up training and testing
trainX, testX, trainY, testY = split_training(dense_table, metadata,
design, training_column,
num_random_test_examples)
model = MultRegression(learning_rate=learning_rate,
clipnorm=clipnorm,
beta_mean=beta_prior,
batch_size=batch_size,
save_path=None)
with tf.Graph().as_default(), tf.Session() as session:
model(session, trainX, trainY, testX, testY)
model.fit(epoch=epoch,
summary_interval=summary_interval,
checkpoint_interval=None)
md_ids = np.array(design.columns)
obs_ids = table.ids(axis='observation')
beta_ = clr(clr_inv(np.hstack((np.zeros((model.p, 1)), model.B))))
beta_ = pd.DataFrame(
beta_.T,
columns=md_ids,
index=obs_ids,
)
return beta_
def ols_regression(output_dir: str, table: pd.DataFrame, tree: skbio.TreeNode,
metadata: Metadata, formula: str) -> None:
if np.any(table.var(axis=0) == 0):
message = ('Detected zero variance balances - '
'double check your table for unobserved features.')
raise UserWarning(message)
res = ols(table=table, metadata=metadata.to_dataframe(), formula=formula)
res.fit()
ols_summary(output_dir, res, tree)
def paired_omics(microbes: biom.Table,
metabolites: biom.Table,
metadata: Metadata = None,
training_column: str = None,
num_testing_examples: int = 5,
min_feature_count: int = 10,
epochs: int = 100,
batch_size: int = 50,
latent_dim: int = 3,
input_prior: float = 1,
output_prior: float = 1,
learning_rate: float = 1e-3,
equalize_biplot: float = False,
arm_the_gpu: bool = False,
summary_interval: int = 60) -> (
pd.DataFrame, OrdinationResults, qiime2.Metadata
):
if metadata is not None:
metadata = metadata.to_dataframe()
if arm_the_gpu:
# pick out the first GPU
device_name = '/device:GPU:0'
else:
device_name = '/cpu:0'
# Note: there are a couple of biom -> pandas conversions taking
# place here. This is currently done on purpose, since we
# haven't figured out how to handle sparse matrix multiplication
# in the context of this algorithm. That is a future consideration.
res = split_tables(
microbes, metabolites,
metadata=metadata, training_column=training_column,
num_test=num_testing_examples,
min_samples=min_feature_count)
(train_microbes_df, test_microbes_df,
train_metabolites_df, test_metabolites_df) = res
train_microbes_coo = coo_matrix(train_microbes_df.values)
test_microbes_coo = coo_matrix(test_microbes_df.values)
with tf.Graph().as_default(), tf.Session() as session:
model = MMvec(
latent_dim=latent_dim,
u_scale=input_prior, v_scale=output_prior,
batch_size=batch_size,
device_name=device_name,
learning_rate=learning_rate)
model(session,
train_microbes_coo, train_metabolites_df.values,
test_microbes_coo, test_metabolites_df.values)
loss, cv = model.fit(epoch=epochs, summary_interval=summary_interval)
ranks = pd.DataFrame(model.ranks(), index=train_microbes_df.columns,
columns=train_metabolites_df.columns)
if latent_dim > 0:
u, s, v = svds(ranks - ranks.mean(axis=0), k=latent_dim)
else:
# fake it until you make it
u, s, v = svds(ranks - ranks.mean(axis=0), k=1)
ranks = ranks.T
ranks.index.name = 'featureid'
s = s[::-1]
u = u[:, ::-1]
v = v[::-1, :]
if equalize_biplot:
microbe_embed = u @ np.sqrt(np.diag(s))
metabolite_embed = v.T @ np.sqrt(np.diag(s))
else:
microbe_embed = u @ np.diag(s)
metabolite_embed = v.T
pc_ids = ['PC%d' % i for i in range(microbe_embed.shape[1])]
features = pd.DataFrame(
microbe_embed, columns=pc_ids,
index=train_microbes_df.columns)
samples = pd.DataFrame(
metabolite_embed, columns=pc_ids,
index=train_metabolites_df.columns)
short_method_name = 'mmvec biplot'
long_method_name = 'Multiomics mmvec biplot'
eigvals = pd.Series(s, index=pc_ids)
proportion_explained = pd.Series(s**2 / np.sum(s**2), index=pc_ids)
biplot = OrdinationResults(
short_method_name, long_method_name, eigvals,
samples=samples, features=features,
proportion_explained=proportion_explained)
its = np.arange(len(loss))
convergence_stats = pd.DataFrame(
{
'loss': loss,
'cross-validation': cv,
'iteration': its
}
)
convergence_stats.index.name = 'id'
convergence_stats.index = convergence_stats.index.astype(np.str)
c = convergence_stats['loss'].astype(np.float)
convergence_stats['loss'] = c
c = convergence_stats['cross-validation'].astype(np.float)
convergence_stats['cross-validation'] = c
c = convergence_stats['iteration'].astype(np.int)
convergence_stats['iteration'] = c
return ranks, biplot, qiime2.Metadata(convergence_stats)
def paired_omics(
microbes: biom.Table,
metabolites: biom.Table,
metadata: Metadata = None,
training_column: str = None,
num_testing_examples: int = 5,
min_feature_count: int = 10,
epochs: int = 100,
batch_size: int = 50,
latent_dim: int = 3,
input_prior: float = 1,
output_prior: float = 1,
learning_rate: float = 0.001,
summary_interval: int = 60) -> (pd.DataFrame, OrdinationResults):
if metadata is not None:
metadata = metadata.to_dataframe()
# Note: there are a couple of biom -> pandas conversions taking
# place here. This is currently done on purpose, since we
# haven't figured out how to handle sparse matrix multiplication
# in the context of this algorithm. That is a future consideration.
res = split_tables(microbes,
metabolites,
metadata=metadata,
training_column=training_column,
num_test=num_testing_examples,
min_samples=min_feature_count)
(train_microbes_df, test_microbes_df, train_metabolites_df,
test_metabolites_df) = res
train_microbes_coo = coo_matrix(train_microbes_df.values)
test_microbes_coo = coo_matrix(test_microbes_df.values)
with tf.Graph().as_default(), tf.Session() as session:
model = MMvec(latent_dim=latent_dim,
u_scale=input_prior,
v_scale=output_prior,
learning_rate=learning_rate)
model(session, train_microbes_coo, train_metabolites_df.values,
test_microbes_coo, test_metabolites_df.values)
loss, cv = model.fit(epoch=epochs, summary_interval=summary_interval)
U, V = model.U, model.V
U_ = np.hstack((np.ones(
(model.U.shape[0], 1)), model.Ubias.reshape(-1, 1), U))
V_ = np.vstack(
(model.Vbias.reshape(1, -1), np.ones((1, model.V.shape[1])), V))
ranks = pd.DataFrame(np.hstack((np.zeros(
(model.U.shape[0], 1)), U_ @ V_)),
index=train_microbes_df.columns,
columns=train_metabolites_df.columns)
ranks = ranks - ranks.mean(axis=1).values.reshape(-1, 1)
ranks = ranks - ranks.mean(axis=0)
u, s, v = svds(ranks, k=latent_dim)
s = s[::-1]
u = u[:, ::-1]
v = v[::-1, :]
microbe_embed = u @ np.diag(s)
metabolite_embed = v.T
pc_ids = ['PC%d' % i for i in range(microbe_embed.shape[1])]
features = pd.DataFrame(microbe_embed,
columns=pc_ids,
index=train_microbes_df.columns)
samples = pd.DataFrame(metabolite_embed,
columns=pc_ids,
index=train_metabolites_df.columns)
short_method_name = 'mmvec biplot'
long_method_name = 'Multiomics mmvec biplot'
eigvals = pd.Series(s, index=pc_ids)
proportion_explained = pd.Series(s**2 / np.sum(s**2), index=pc_ids)
biplot = OrdinationResults(short_method_name,
long_method_name,
eigvals,
samples=samples,
features=features,
proportion_explained=proportion_explained)
return ranks, biplot
def _load_metadata(metadata: Metadata = None):
if not metadata:
raise ValueError('Metadata parameter not provided!')
metadata = metadata.to_dataframe()
return metadata
def multinomial(table: biom.Table,
metadata: Metadata,
formula: str,
training_column: str = DEFAULTS["training-column"],
num_random_test_examples: int = (
DEFAULTS["num-random-test-examples"]
),
epochs: int = DEFAULTS["epochs"],
batch_size: int = DEFAULTS["batch-size"],
differential_prior: float = DEFAULTS["differential-prior"],
learning_rate: float = DEFAULTS["learning-rate"],
clipnorm: float = DEFAULTS["clipnorm"],
min_sample_count: int = DEFAULTS["min-sample-count"],
min_feature_count: int = DEFAULTS["min-feature-count"],
summary_interval: int = DEFAULTS["summary-interval"],
random_seed: int = DEFAULTS["random-seed"],
) -> (
pd.DataFrame, qiime2.Metadata, skbio.OrdinationResults
):
# load metadata and tables
metadata = metadata.to_dataframe()
# match them
table, metadata, design = match_and_filter(
table, metadata,
formula, min_sample_count, min_feature_count
)
# convert to dense representation
dense_table = table.to_dataframe().to_dense().T
# split up training and testing
trainX, testX, trainY, testY = split_training(
dense_table, metadata, design,
training_column, num_random_test_examples,
seed=random_seed,
)
model = MultRegression(learning_rate=learning_rate, clipnorm=clipnorm,
beta_mean=differential_prior,
batch_size=batch_size,
save_path=None)
with tf.Graph().as_default(), tf.Session() as session:
tf.set_random_seed(random_seed)
model(session, trainX, trainY, testX, testY)
loss, cv, its = model.fit(
epochs=epochs,
summary_interval=summary_interval,
checkpoint_interval=None)
md_ids = np.array(design.columns)
obs_ids = table.ids(axis='observation')
beta_ = np.hstack((np.zeros((model.p, 1)), model.B))
beta_ = beta_ - beta_.mean(axis=1).reshape(-1, 1)
differentials = pd.DataFrame(
beta_.T, columns=md_ids, index=obs_ids,
)
differentials.index.name = 'featureid'
convergence_stats = pd.DataFrame(
{
'loss': loss,
'cross-validation': cv,
'iteration': its
}
)
convergence_stats.index.name = 'id'
convergence_stats.index = convergence_stats.index.astype(np.str)
c = convergence_stats['loss'].astype(np.float)
convergence_stats['loss'] = c
c = convergence_stats['cross-validation'].astype(np.float)
convergence_stats['cross-validation'] = c
c = convergence_stats['iteration'].astype(np.int)
convergence_stats['iteration'] = c
# regression biplot
if differentials.shape[-1] > 1:
u, s, v = np.linalg.svd(differentials)
pc_ids = ['PC%d' % i for i in range(len(s))]
samples = pd.DataFrame(u[:, :len(s)] @ np.diag(s),
columns=pc_ids, index=differentials.index)
features = pd.DataFrame(v.T[:, :len(s)],
columns=pc_ids, index=differentials.columns)
short_method_name = 'regression_biplot'
long_method_name = 'Multinomial regression biplot'
eigvals = pd.Series(s, index=pc_ids)
proportion_explained = eigvals**2 / (eigvals**2).sum()
biplot = OrdinationResults(
short_method_name, long_method_name, eigvals,
samples=samples, features=features,
proportion_explained=proportion_explained)
else:
# this is to handle the edge case with only intercepts
biplot = OrdinationResults('', '', pd.Series(), pd.DataFrame())
return differentials, qiime2.Metadata(convergence_stats), biplot
def _2(ff: MetadataFormat) -> Metadata:
path = str(ff) + '/metadata.tsv'
return Metadata.load(path)
def _1(data: Metadata) -> MetadataFormat:
ff = MetadataFormat()
path = str(ff) + '/metadata.tsv'
data.save(path)
return ff