def test_to_dict_with_feature_metadata(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
self.feature_metadata,
filter_unobserved_features_from_phylogeny=False)
obs = viz._to_dict()
dict_a_with_fm = copy.deepcopy(DICT_A)
dict_a_with_fm["tip_metadata"] = {
"a": {
"fmdcol1": "asdf",
"fmdcol2": "qwer"
}
}
dict_a_with_fm["int_metadata"] = {
"h": {
"fmdcol1": "ghjk",
"fmdcol2": "tyui"
}
}
dict_a_with_fm["feature_metadata_columns"] = ["fmdcol1", "fmdcol2"]
tree_data = obs['tree_data']
exp = dict_a_with_fm['tree_data']
self.assert_almost_equal_tree_data(tree_data, exp)
obs.pop('tree_data')
dict_a_with_fm.pop('tree_data')
self.assertEqual(obs, dict_a_with_fm)
def test_to_dict_with_metadata_nans(self):
nan_sample_metadata = self.sample_metadata.copy()
nan_feature_metadata = self.feature_metadata.copy()
nan_sample_metadata.at["Sample2", "Metadata4"] = np.nan
nan_feature_metadata.at["h", "fmdcol1"] = np.nan
nan_feature_metadata.at["a", "fmdcol2"] = np.nan
viz = Empress(self.tree,
self.table,
nan_sample_metadata,
nan_feature_metadata,
shear_to_table=False)
obs = viz.to_dict()
dict_a_nan = copy.deepcopy(DICT_A)
# [1][3] corresponds to Sample2, Metadata4
dict_a_nan["compressed_sample_metadata"][1][3] = str(np.nan)
dict_a_nan["compressed_tip_metadata"] = {1: ["asdf", str(np.nan)]}
dict_a_nan["compressed_int_metadata"] = {8: [str(np.nan), "tyui"]}
dict_a_nan["feature_metadata_columns"] = ["fmdcol1", "fmdcol2"]
self.assertEqual(obs, dict_a_nan)
res = viz.make_empress()
self.assertTrue('empressRequire' in res)
self.assertTrue('empress = new Empress' in res)
self.assertTrue('emperor_require_logic' not in res)
def test_ordination_integration_callbacks(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.pcoa)
# table should be unchanged and be a different id instance
self.assertEqual(self.table, viz.table)
self.assertNotEqual(id(self.table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.sample_metadata, viz.samples)
self.assertNotEqual(id(self.sample_metadata), id(viz.samples))
self.assertIsNone(viz.features)
assert_ordination_results_equal(viz.ordination, self.pcoa)
# emperor is instantiated as needed but not yet setup
self.assertTrue(isinstance(viz._emperor, Emperor))
# ensure the callbacks were rendered
obs = viz.make_empress()
self.assertTrue('setOnNodeMenuVisibleCallback' in obs)
self.assertTrue('setOnNodeMenuHiddenCallback' in obs)
def test_init_only_one_of_table_and_sm_passed(self):
exp_errmsg = (
"Both the table and sample metadata should be specified or None. "
"However, only one of them is None.")
with self.assertRaisesRegex(ValueError, exp_errmsg):
Empress(self.tree, self.table)
with self.assertRaisesRegex(ValueError, exp_errmsg):
Empress(self.tree, sample_metadata=self.sample_metadata)
def test_to_dict_tree_plot(self):
viz = Empress(self.tree)
dict_a_cp = copy.deepcopy(DICT_A)
self._clear_copied_dict_a(dict_a_cp)
obs = viz.to_dict()
self.assertEqual(obs, dict_a_cp)
def test_init_tree_plot(self):
# Simplest case (no feature metadata)
viz = Empress(self.tree)
self.assertFalse(viz.is_community_plot)
self.assertIsNone(viz.tip_md)
self.assertIsNone(viz.int_md)
# Slightly less simple case (with feature metadata)
viz = Empress(self.tree, feature_metadata=self.feature_metadata)
self.assertFalse(viz.is_community_plot)
assert_frame_equal(viz.tip_md, self.feature_metadata.loc[["a"]])
assert_frame_equal(viz.int_md, self.feature_metadata.loc[["h"]])
def test_to_dict_with_feature_metadata(self):
viz = Empress(
self.tree, self.table, self.sample_metadata, self.feature_metadata,
shear_to_table=False
)
obs = viz.to_dict()
dict_a_with_fm = copy.deepcopy(DICT_A)
dict_a_with_fm["compressed_tip_metadata"] = {1: ["asdf", "qwer"]}
dict_a_with_fm["compressed_int_metadata"] = {8: ["ghjk", "tyui"]}
dict_a_with_fm["feature_metadata_columns"] = ["fmdcol1", "fmdcol2"]
self.assertEqual(obs, dict_a_with_fm)
def test_copy_support_files_use_base(self):
local_path = './some-local-path/'
viz = Empress(self.tree, self.table, self.sample_metadata,
resource_path=local_path,
shear_to_table=False)
self.assertEqual(viz.base_url, local_path)
viz.copy_support_files()
self.assertTrue(exists(local_path))
self.files_to_remove.append(local_path)
def test_to_dict_tree_plot_with_feature_metadata(self):
viz = Empress(self.tree, feature_metadata=self.feature_metadata)
# Set up expected dict
dict_a_cp = copy.deepcopy(DICT_A)
self._clear_copied_dict_a(dict_a_cp)
# Copied from test_to_dict_with_feature_metadata() above
dict_a_cp["compressed_tip_metadata"] = {1: ["asdf", "qwer"]}
dict_a_cp["compressed_int_metadata"] = {8: ["ghjk", "tyui"]}
dict_a_cp["feature_metadata_columns"] = ["fmdcol1", "fmdcol2"]
obs = viz.to_dict()
self.assertEqual(obs, dict_a_cp)
def test_no_intersection_between_tree_and_table(self):
with self.assertRaisesRegex(
tools.DataMatchingError,
"No features in the feature table are present as tips in the tree."
):
Empress(self.tree, self.unrelated_table, self.sample_metadata,
shear_to_table=False)
# Check that --p-shear-to-table doesn't override this: the data
# mismatch should be identified before attempting shearing
with self.assertRaisesRegex(
tools.DataMatchingError,
"No features in the feature table are present as tips in the tree."
):
Empress(self.tree, self.unrelated_table, self.sample_metadata,
shear_to_table=True)
def test_copy_support_files_use_base(self):
local_path = './some-local-path/'
viz = Empress(self.tree,
self.table,
self.sample_metadata,
resource_path=local_path,
filter_unobserved_features_from_phylogeny=False)
self.assertEqual(viz.base_url, local_path)
viz.copy_support_files()
self.assertTrue(exists(local_path))
self.files_to_remove.append(local_path)
def test_init(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
shear_to_table=False)
self.assertEqual(viz.base_url, 'support_files')
self.assertEqual(
list(viz.tree.B),
[1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0])
names = ['a', 'e', None, 'b', 'g', None, 'd', 'h', None]
for i in range(1, len(viz.tree) + 1):
node = viz.tree.postorderselect(i)
self.assertEqual(viz.tree.name(node), names[i - 1])
# table should be unchanged and be a different id instance
self.assertEqual(self.table, viz.table)
self.assertNotEqual(id(self.table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.sample_metadata, viz.samples)
self.assertNotEqual(id(self.sample_metadata), id(viz.samples))
self.assertIsNone(viz.features)
self.assertIsNone(viz.ordination)
self.assertTrue(viz.is_community_plot)
def community_plot(
tree: str,
table: str,
sample_metadata: str,
output_dir: str,
pcoa: str,
feature_metadata: str,
ignore_missing_samples: bool,
filter_extra_samples: bool,
filter_missing_features: bool,
number_of_pcoa_features: int,
shear_to_table: bool,
) -> None:
tree_newick, fm = check_and_process_files(output_dir, tree,
feature_metadata)
table = load_table(table)
sample_metadata = pd.read_csv(sample_metadata, sep="\t", index_col=0)
if pcoa is not None:
pcoa = OrdinationResults.read(pcoa)
pcoa = prepare_pcoa(pcoa, number_of_pcoa_features)
viz = Empress(
tree_newick,
table=table,
sample_metadata=sample_metadata,
feature_metadata=fm,
ordination=pcoa,
ignore_missing_samples=ignore_missing_samples,
filter_extra_samples=filter_extra_samples,
filter_missing_features=filter_missing_features,
shear_to_table=shear_to_table,
)
os.makedirs(output_dir)
save_viz(viz, output_dir, q2=False)
def test_init(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
filter_unobserved_features_from_phylogeny=False)
self.assertEqual(viz.base_url, './')
self.assertEqual(
viz._bp_tree,
[1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0])
names = [
'a', 'e', 'EmpressNode0', 'b', 'g', 'EmpressNode1', 'd', 'h',
'EmpressNode2'
]
for i, node in enumerate(viz.tree.postorder()):
self.assertEqual(node.name, names[i])
# table should be unchanged and be a different id instance
assert_frame_equal(self.table, viz.table.T)
self.assertNotEqual(id(self.table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.sample_metadata, viz.samples)
self.assertNotEqual(id(self.sample_metadata), id(viz.samples))
self.assertIsNone(viz.features)
self.assertIsNone(viz.ordination)
def test_init_with_ordination(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.pcoa,
shear_to_table=False)
self.assertEqual(viz.base_url, 'support_files')
self.assertEqual(
list(viz.tree.B),
[1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0])
names = ['a', 'e', None, 'b', 'g', None, 'd', 'h', None]
for i in range(1, len(viz.tree) + 1):
node = viz.tree.postorderselect(i)
self.assertEqual(viz.tree.name(node), names[i - 1])
# table should be unchanged and be a different id instance
self.assertEqual(self.table, viz.table)
self.assertNotEqual(id(self.table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.sample_metadata, viz.samples)
self.assertNotEqual(id(self.sample_metadata), id(viz.samples))
self.assertIsNone(viz.features)
assert_ordination_results_equal(viz.ordination, self.pcoa)
# emperor is instantiated as needed but not yet setup
self.assertTrue(isinstance(viz._emperor, Emperor))
def test_fm_filtering_post_shearing(self):
extra_fm = self.feature_metadata.copy()
extra_fm.loc["e"] = "i'm going to be filtered :O"
viz = Empress(self.tree,
self.filtered_table,
self.filtered_sample_metadata,
feature_metadata=extra_fm,
shear_to_table=True)
# Same as with the shearing test above, check that the tree was handled
# as expected
self.assertEqual(list(viz.tree.B),
[1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0])
names = ['a', None, 'b', 'g', 'd', 'h', None]
for i in range(1, len(viz.tree) + 1):
node = viz.tree.postorderselect(i)
self.assertEqual(viz.tree.name(node), names[i - 1])
# Now, the point of this test: verify that the feature metadata was
# filtered to just stuff in the sheared tree ("e" was removed from the
# tip metadata)
assert_frame_equal(extra_fm.loc[["a"]], viz.tip_md)
assert_frame_equal(extra_fm.loc[["h"]], viz.int_md)
# table should be unchanged and be a different id instance
self.assertEqual(self.filtered_table, viz.table)
self.assertNotEqual(id(self.filtered_table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.filtered_sample_metadata, viz.samples)
self.assertNotEqual(id(self.filtered_sample_metadata), id(viz.samples))
self.assertIsNone(viz.ordination)
def test_shear_tree_to_fm_simple(self):
# remove e same as in test_shear_tree
mini_fm = self.feature_metadata.copy()
mini_fm.loc["b"] = ["pikachu", "raichu"]
mini_fm.loc["d"] = ["mew", "mewtwo"]
viz = Empress(self.tree,
feature_metadata=mini_fm,
shear_to_table=False,
shear_to_feature_metadata=True)
self.assertEqual(list(viz.tree.B),
[1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0])
names = ['a', None, 'b', 'g', 'd', 'h', None]
for i in range(1, len(viz.tree) + 1):
node = viz.tree.postorderselect(i)
self.assertEqual(viz.tree.name(node), names[i - 1])
assert_frame_equal(viz.tip_md, mini_fm.loc[["a", "b", "d"]])
assert_frame_equal(viz.int_md, mini_fm.loc[["h"]])
# feature metadata should be unchanged and be a different id instance
assert_frame_equal(mini_fm, viz.features)
self.assertNotEqual(id(mini_fm), id(viz.features))
self.assertIsNone(viz.ordination)
def test_fm_filtering_post_shearing_with_moving_pictures_dataset(self):
# This particular tip can be problematic (it was the reason we found
# out about https://github.com/biocore/empress/issues/248), so we
# observe how it is handled in generating a visualization of the
# moving pictures dataset to verify that #248 does not recur.
funky_tip = "8406abe6d9a72018bf32d189d1340472"
tree, tbl, smd, fmd, pcoa = load_mp_data()
# Convert artifacts / metadata objects to "normal" types that we can
# pass to Empress
bp_tree = from_skbio_treenode(tree.view(TreeNode))
tbl_df = tbl.view(biom.Table)
pcoa_skbio = pcoa.view(skbio.OrdinationResults)
smd_df = smd.to_dataframe()
fmd_df = fmd.to_dataframe()
# Sanity check -- verify that the funky tip we're looking for is
# actually present in the data. (We haven't actually done anything
# specific to Empress yet. This just verifies the environment is ok.)
# https://stackoverflow.com/a/23549599/10730311
self.assertTrue(funky_tip in fmd_df.index)
# Generate an Empress visualization using this data
viz = Empress(bp_tree,
tbl_df,
smd_df,
feature_metadata=fmd_df,
ordination=pcoa_skbio,
filter_extra_samples=True,
shear_to_table=True)
# Check that tip 8406abe6d9a72018bf32d189d1340472 *isn't* in the tip
# metadata. All of the samples this tip is present in are filtered out
# when --p-filter-extra-samples is used with this particular PCoA, so
# we verify that this tip is removed from the tip metadata.
self.assertFalse(funky_tip in viz.tip_md.index)
def test_init_with_ordination(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.pcoa,
filter_unobserved_features_from_phylogeny=False)
self.assertEqual(viz.base_url, './')
self.assertEqual(
viz._bp_tree,
[1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0])
names = [
'a', 'e', 'EmpressNode0', 'b', 'g', 'EmpressNode1', 'd', 'h',
'EmpressNode2'
]
for i, node in enumerate(viz.tree.postorder()):
self.assertEqual(node.name, names[i])
# table should be unchanged and be a different id instance
assert_frame_equal(self.table, viz.table.T)
self.assertNotEqual(id(self.table), id(viz.table))
# sample metadata should be unchanged and be a different id instance
assert_frame_equal(self.sample_metadata, viz.samples)
self.assertNotEqual(id(self.sample_metadata), id(viz.samples))
self.assertIsNone(viz.features)
assert_ordination_results_equal(viz.ordination, self.pcoa)
# emperor is instantiated as needed but not yet setup
self.assertTrue(isinstance(viz._emperor, Emperor))
def test_to_dict(self):
viz = Empress(self.tree, self.table, self.sample_metadata,
shear_to_table=False)
obs = viz.to_dict()
dict_a_cp = copy.deepcopy(DICT_A)
# NOTE: Uncomment the following two lines of code to write the current
# DICT_A to a file. Once it's written to a file, you can run
# "black -l 79 dictcode.py" (while in the same directory as the file)
# to format it so that it's consistent with how DICT_A is set up at the
# bottom of this file.
# with open("dictcode.py", "w") as f:
# f.write("DICT_A = {}".format(str(obs)))
self.assertEqual(obs, dict_a_cp)
def test_to_dict(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
filter_unobserved_features_from_phylogeny=False)
obs = viz._to_dict()
dict_a_cp = copy.deepcopy(DICT_A)
tree_data = obs['tree_data']
exp = dict_a_cp['tree_data']
self.assert_almost_equal_tree_data(tree_data, exp)
dict_a_cp.pop('tree_data')
obs.pop('tree_data')
self.assertEqual(obs, dict_a_cp)
def plot(output_dir: str,
tree: NewickFormat,
feature_table: pd.DataFrame,
sample_metadata: qiime2.Metadata,
pcoa: OrdinationResults = None,
feature_metadata: qiime2.Metadata = None,
ignore_missing_samples: bool = False,
filter_missing_features: bool = False,
number_of_features: int = 5,
filter_unobserved_features_from_phylogeny: bool = True) -> None:
if pcoa is not None and pcoa.features is not None:
# select the top N most important features based on the vector's
# magnitude (coped from q2-emperor)
feats = pcoa.features.copy()
origin = np.zeros_like(feats.columns)
feats['importance'] = feats.apply(euclidean, axis=1, args=(origin, ))
feats.sort_values('importance', inplace=True, ascending=False)
feats.drop(['importance'], inplace=True, axis=1)
pcoa.features = feats[:number_of_features].copy()
sample_metadata = sample_metadata.to_dataframe()
if feature_metadata is not None:
feature_metadata = feature_metadata.to_dataframe()
# path to the actual newick file
with open(str(tree)) as file:
t = parse_newick(file.readline())
trim_tree = filter_unobserved_features_from_phylogeny
viz = Empress(tree=t,
table=feature_table,
sample_metadata=sample_metadata,
feature_metadata=feature_metadata,
ordination=pcoa,
ignore_missing_samples=ignore_missing_samples,
filter_missing_features=filter_missing_features,
filter_unobserved_features_from_phylogeny=trim_tree)
with open(os.path.join(output_dir, 'empress.html'), 'w') as file:
file.write(str(viz))
viz.copy_support_files(output_dir)
index = os.path.join(TEMPLATES, 'index.html')
q2templates.render(index, output_dir)
def test_init_with_ordination_features(self):
'''Check that empress does not break when ordination has features
but empress itself does not.'''
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.biplot,
shear_to_table=False)
self.assertIsNone(viz.features)
def test_biplot_no_matching(self):
self.feature_metadata.index = ['z', 'y']
viz = Empress(self.biplot_tree, self.biplot_table,
self.sample_metadata,
feature_metadata=self.feature_metadata,
ordination=self.biplot_no_matches,
shear_to_table=True)
obs = str(viz)
self.assertTrue('All elements' in obs)
def test_to_dict_with_emperor(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.pcoa,
filter_unobserved_features_from_phylogeny=False)
obs = viz._to_dict()
self.assertEqual(viz._emperor.width, '48vw')
self.assertEqual(viz._emperor.height, '100vh; float: right')
self.assertEqual(viz._emperor.settings['axes']['axesColor'], 'black')
self.assertEqual(viz._emperor.settings['axes']['backgroundColor'],
'white')
# we test key by key so we can do "general" checks on the emperor
# values, this helps with tests not breaking if any character changes
# in # Emperor
for key, value in obs.items():
if key == 'tree_data':
tree_data = obs['tree_data']
exp = DICT_A['tree_data']
self.assert_almost_equal_tree_data(tree_data, exp)
elif not key.startswith('emperor_'):
self.assertEqual(obs[key], DICT_A[key])
exp = " <div id='emperor-notebook"
self.assertTrue(obs['emperor_div'].startswith(exp))
exp = "// When running in the Jupyter"
self.assertTrue(obs['emperor_require_logic'].startswith(exp))
exp = "}); // END REQUIRE.JS block"
self.assertTrue(obs['emperor_require_logic'].endswith(exp))
exp = '<link id="emperor-css" rel="stylesheet"'
self.assertTrue(obs['emperor_style'].startswith(exp))
exp = "vendor/js/jquery-"
self.assertEqual(obs['emperor_base_dependencies'].count(exp), 1)
self.assertTrue(obs['emperor_classes'], 'combined-plot-container')
def test_to_dict_with_emperor(self):
viz = Empress(self.tree,
self.table,
self.sample_metadata,
ordination=self.pcoa,
shear_to_table=False,
filter_extra_samples=True)
obs = viz.to_dict()
self.assertEqual(viz._emperor.width, '50vw')
self.assertEqual(viz._emperor.height, '100vh; float: right')
self.assertEqual(viz._emperor.settings['axes']['axesColor'], 'black')
self.assertEqual(viz._emperor.settings['axes']['backgroundColor'],
'white')
# we test key by key so we can do "general" checks on the emperor
# values, this helps with tests not breaking if any character changes
# in # Emperor
dict_a_cp = copy.deepcopy(DICT_A)
# set is_empire_plot flag to True since DICT_A sets it as False (all
# other test use a False value)
dict_a_cp["is_empire_plot"] = True
for key, value in obs.items():
if not key.startswith('emperor_'):
self.assertEqual(obs[key], dict_a_cp[key])
exp = "<div id='emperor-in-empire'"
self.assertTrue(obs['emperor_div'].startswith(exp))
exp = "// When running in the Jupyter"
self.assertTrue(obs['emperor_require_logic'].startswith(exp))
exp = "}); // END REQUIRE.JS block"
self.assertTrue(obs['emperor_require_logic'].endswith(exp))
self.assertTrue('"#emperor-css"' in obs['emperor_style'])
exp = "vendor/js/jquery-"
self.assertEqual(obs['emperor_base_dependencies'].count(exp), 1)
self.assertTrue(obs['emperor_classes'], 'combined-plot-container')
def tree_plot(output_dir: str,
tree: NewickFormat,
feature_metadata: qiime2.Metadata = None) -> None:
"""Visualizes a tree (optionally with feature metadata)."""
if feature_metadata is not None:
feature_metadata = feature_metadata.to_dataframe()
t = get_bp(tree)
viz = Empress(tree=t, feature_metadata=feature_metadata)
save_viz(viz, output_dir)
def test_init_tree_plot_fm_not_matching(self):
# Mainly, this test validates that the matching done between the tree
# nodes and feature metadata is still performed even if tree-plot is
# used.
bad_fm = self.feature_metadata.copy()
bad_fm.index = ["idont", "match :O"]
with self.assertRaisesRegex(
tools.DataMatchingError,
("No features in the feature metadata are present in the tree, "
"either as tips or as internal nodes.")):
Empress(self.tree, feature_metadata=bad_fm)
def test_no_intersection_between_tree_and_table(self):
bad_table = self.unrelated_table.copy()
bad_table.index = range(len(self.unrelated_table.index))
with self.assertRaisesRegex(
tools.DataMatchingError,
"No features in the feature table are present as tips in the tree."
):
Empress(self.tree,
self.unrelated_table,
self.sample_metadata,
filter_unobserved_features_from_phylogeny=False)
# Check that --p-filter-unobserved-features-from-phylogeny doesn't
# override this: the data mismatch should be identified before
# attempting shearing
with self.assertRaisesRegex(
tools.DataMatchingError,
"No features in the feature table are present as tips in the tree."
):
Empress(self.tree,
self.unrelated_table,
self.sample_metadata,
filter_unobserved_features_from_phylogeny=True)
def test_biplot_partial_match_override(self):
fm = self.feature_metadata.copy()
fm.index = ['a', 'x']
viz = Empress(self.tree, self.table,
self.sample_metadata,
feature_metadata=fm,
ordination=self.biplot,
ignore_missing_samples=True,
shear_to_table=True)
obs = str(viz)
self.assertTrue('This element has no metadata' in obs)