def test_evolutionary_coupling_custom_dissimilarity_map(self):
weight_matrix = pd.DataFrame.from_dict(
{
"D": [0.0, 0.5, 1.2, 0.4, 0.5, 0.6],
"F": [0.5, 0.0, 3.0, 1.1, 3.0, 0.1],
"G": [1.2, 3.0, 0.0, 0.8, 0.2, 0.8],
"H": [0.4, 1.1, 0.8, 0.0, 2.0, 2.1],
"I": [0.5, 3.0, 0.2, 2.0, 0.0, 0.1],
"J": [0.6, 0.1, 1.8, 2.1, 0.1, 0.0],
},
orient="index",
columns=["D", "F", "G", "H", "I", "J"],
)
random_state = np.random.RandomState(1231234)
evolutionary_coupling = cas.tl.compute_evolutionary_coupling(
self.tree,
meta_variable="CellType",
random_state=random_state,
minimum_proportion=0.0,
number_of_shuffles=10,
dissimilarity_map=weight_matrix,
)
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
self.tree, meta_item="CellType", dissimilarity_map=weight_matrix)
# background computed with random seed set above and 10 shuffles
# (state1, state2): (mean, sd)
expected_summary_stats = {
("TypeB", "TypeB"): (0.695, 0.5456418239101545),
("TypeB", "TypeA"): (1.0000000000000002, 0.281291663580704),
("TypeB", "TypeC"): (1.0925, 0.44763964301656745),
("TypeA", "TypeB"): (1.0000000000000002, 0.3148412298286232),
("TypeA", "TypeA"): (0.63, 0.4550824101193101),
("TypeA", "TypeC"): (1.2349999999999999, 0.391503512117069),
("TypeC", "TypeB"): (1.0675000000000001, 0.4493119740225047),
("TypeC", "TypeA"): (1.26, 0.41791147387933725),
("TypeC", "TypeC"): (0.4699999999999999, 0.41424630354415953),
}
expected_coupling = inter_cluster_distances.copy()
for s1 in expected_coupling.index:
for s2 in expected_coupling.columns:
mean = expected_summary_stats[(s1, s2)][0]
sd = expected_summary_stats[(s1, s2)][1]
expected_coupling.loc[
s1, s2] = (inter_cluster_distances.loc[s1, s2] - mean) / sd
pd.testing.assert_frame_equal(expected_coupling,
evolutionary_coupling,
atol=0.001)
def test_inter_cluster_distance_custom_input(self):
tree = nx.DiGraph()
tree.add_nodes_from(["A", "B", "C", "D", "E", "F"])
tree.add_edge("F", "A", length=0.1)
tree.add_edge("F", "B", length=0.2)
tree.add_edge("F", "E", length=0.5)
tree.add_edge("E", "C", length=0.3)
tree.add_edge("E", "D", length=0.4)
meta_data = pd.DataFrame.from_dict(
{
"A": ["TypeA", 10],
"B": ["TypeA", 5],
"C": ["TypeB", 3],
"D": ["TypeB", 22],
},
orient="index",
columns=["CellType", "nUMI"],
)
weight_matrix = pd.DataFrame.from_dict(
{
"A": [0.0, 0.5, 1.2, 0.4],
"B": [0.5, 0.0, 3.0, 1.1],
"C": [1.2, 3.0, 0.0, 0.8],
"D": [0.4, 1.1, 0.8, 0.0],
},
orient="index",
columns=["A", "B", "C", "D"],
)
tree = CassiopeiaTree(tree=tree)
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
tree,
meta_data=meta_data["CellType"],
dissimilarity_map=weight_matrix,
)
expected_distances = pd.DataFrame.from_dict(
{
"TypeA": [0.25, 1.425],
"TypeB": [1.425, 0.4]
},
orient="index",
columns=["TypeA", "TypeB"],
)
pd.testing.assert_frame_equal(
expected_distances,
inter_cluster_distances,
check_exact=False,
atol=0.001,
)
def test_evolutionary_coupling_basic(self):
random_state = np.random.RandomState(1231234)
evolutionary_coupling = cas.tl.compute_evolutionary_coupling(
self.tree,
meta_variable="CellType",
random_state=random_state,
minimum_proportion=0.0,
number_of_shuffles=10,
)
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
self.tree, meta_item="CellType")
# background computed with random seed set above and 10 shuffles
# (state1, state2): (mean, sd)
expected_summary_stats = {
("TypeA", "TypeA"): (1.7, 0.6000000000000001),
("TypeA", "TypeB"): (3.55, 0.4716990566028302),
("TypeA", "TypeC"): (3.55, 0.4716990566028302),
("TypeB", "TypeA"): (3.55, 0.4716990566028302),
("TypeB", "TypeB"): (2.0, 0.5),
("TypeB", "TypeC"): (3.65, 0.45),
("TypeC", "TypeA"): (3.55, 0.4716990566028302),
("TypeC", "TypeB"): (3.65, 0.45),
("TypeC", "TypeC"): (1.8, 0.5567764362830022),
}
expected_coupling = inter_cluster_distances.copy()
for s1 in expected_coupling.index:
for s2 in expected_coupling.columns:
mean = expected_summary_stats[(s1, s2)][0]
sd = expected_summary_stats[(s1, s2)][1]
expected_coupling.loc[
s1, s2] = (inter_cluster_distances.loc[s1, s2] - mean) / sd
pd.testing.assert_frame_equal(expected_coupling,
evolutionary_coupling,
atol=0.001)
# make sure errors are raised for numerical data
self.assertRaises(
CassiopeiaError,
cas.tl.compute_evolutionary_coupling,
self.tree,
"nUMI",
)
def test_evolutionary_coupling_minimum_proportion(self):
self.tree.cell_meta.loc["J", "CellType"] = "TypeD"
random_state = np.random.RandomState(1231234)
evolutionary_coupling = cas.tl.compute_evolutionary_coupling(
self.tree,
meta_variable="CellType",
random_state=random_state,
minimum_proportion=1 / 6, # This will drop types C and D
number_of_shuffles=10,
)
# make sure TypeC and TypeD are not in the evolutionary coupling matrix
expected_types = ["TypeA", "TypeB"]
self.assertCountEqual(expected_types, evolutionary_coupling.index)
self.assertCountEqual(expected_types, evolutionary_coupling.columns)
# make sure couplings are correct
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
self.tree, meta_item="CellType")
inter_cluster_distances = inter_cluster_distances.loc[expected_types,
expected_types]
expected_summary_stats = {
("TypeB", "TypeB"): (1.4, 0.19999999999999998),
("TypeB", "TypeA"): (2.6, 0.19999999999999998),
("TypeA", "TypeB"): (2.6, 0.19999999999999998),
("TypeA", "TypeA"): (1.4, 0.19999999999999998),
}
expected_coupling = inter_cluster_distances.copy()
for s1 in expected_coupling.index:
for s2 in expected_coupling.columns:
mean = expected_summary_stats[(s1, s2)][0]
sd = expected_summary_stats[(s1, s2)][1]
expected_coupling.loc[
s1, s2] = (inter_cluster_distances.loc[s1, s2] - mean) / sd
evolutionary_coupling = evolutionary_coupling.loc[expected_types,
expected_types]
pd.testing.assert_frame_equal(expected_coupling,
evolutionary_coupling,
atol=0.001)
def test_inter_cluster_distance_basic(self):
tree = nx.DiGraph()
tree.add_nodes_from(["A", "B", "C", "D", "E", "F"])
tree.add_edge("F", "A", length=0.1)
tree.add_edge("F", "B", length=0.2)
tree.add_edge("F", "E", length=0.5)
tree.add_edge("E", "C", length=0.3)
tree.add_edge("E", "D", length=0.4)
meta_data = pd.DataFrame.from_dict(
{
"A": ["TypeA", 10],
"B": ["TypeA", 5],
"C": ["TypeB", 3],
"D": ["TypeB", 22],
},
orient="index",
columns=["CellType", "nUMI"],
)
tree = CassiopeiaTree(tree=tree, cell_meta=meta_data)
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
tree, meta_item="CellType")
expected_distances = pd.DataFrame.from_dict(
{
"TypeA": [0.15, 1.0],
"TypeB": [1.0, 0.35]
},
orient="index",
columns=["TypeA", "TypeB"],
)
pd.testing.assert_frame_equal(expected_distances,
inter_cluster_distances)
self.assertRaises(
CassiopeiaError,
data_utilities.compute_inter_cluster_distances,
tree,
"nUMI",
)
def compute_evolutionary_coupling(
tree: CassiopeiaTree,
meta_variable: str,
minimum_proportion: float = 0.05,
number_of_shuffles: int = 500,
random_state: Optional[np.random.RandomState] = None,
dissimilarity_map: Optional[pd.DataFrame] = None,
cluster_comparison_function: Callable = data_utilities.
net_relatedness_index,
**comparison_kwargs,
) -> pd.DataFrame:
"""Computes Evolutionary Coupling of categorical variables.
Using the methodology described in Yang, Jones et al, BioRxiv (2021), this
function will compute the "evolutionary coupling" statistic between values
that a categorical variable can take on with the tree. For example, this
categorical variable can be a "cell type", and this function will compute
the evolutionary couplings between all types of cell types. This indicates
how closely related these cell types are to one another.
Briefly, this statistic is the Z-normalized mean distance between categories
in the specified categorical variable. Note that empirical nulls that have a
standard deviation of 0 lead to NaNs in the resulting evolutionary coupling
matrix.
The computational complexity of this function is
O(n^2 log n + (B+1)(K^2 * O(distance_function)) for a tree with n leaves, a
variable with K categories, and B random shuffles.
Args:
tree: CassiopeiaTree
meta_variable: Column in `tree.cell_meta` that stores a categorical
variable with K categories.
minimum_proportion: Minimum proportion of cells that a category needs
to appear in to be considered.
number_of_shuffles: Number of times to shuffle the data to compute the
empirical Z score.
random_state: Numpy random state to parameterize the shuffling.
dissimilarity_map: A precomputed dissimilarity map between all leaves.
cluster_comparison_function: A function for comparing the mean distance
between groups. By default, this is the Net Relatedness Index.
**comparison_kwargs: Extra arguments to pass to the cluster comparison
function.
Returns:
A K x K evolutionary coupling dataframe.
"""
W = (data_utilities.compute_phylogenetic_weight_matrix(tree) if
(dissimilarity_map is None) else dissimilarity_map)
meta_data = tree.cell_meta[meta_variable]
# subset meta data by minimum proportion
if minimum_proportion > 0:
filter_threshold = int(len(tree.leaves) * minimum_proportion)
category_frequencies = meta_data.value_counts()
passing_categories = category_frequencies[
category_frequencies > filter_threshold].index.values
meta_data = meta_data[meta_data.isin(passing_categories)]
W = W.loc[meta_data.index.values, meta_data.index.values]
# compute inter-cluster distances
inter_cluster_distances = data_utilities.compute_inter_cluster_distances(
tree,
meta_data=meta_data,
dissimilarity_map=W,
distance_function=cluster_comparison_function,
**comparison_kwargs,
)
# compute background for Z-scoring
background = defaultdict(list)
for _ in tqdm(range(number_of_shuffles),
desc="Creating empirical background"):
permuted_assignments = meta_data.copy()
if random_state:
permuted_assignments.index = random_state.permutation(
meta_data.index.values)
else:
permuted_assignments.index = np.random.permutation(
meta_data.index.values)
background_distances = data_utilities.compute_inter_cluster_distances(
tree,
meta_data=permuted_assignments,
dissimilarity_map=W,
distance_function=cluster_comparison_function,
**comparison_kwargs,
)
for s1 in background_distances.index:
for s2 in background_distances.columns:
background[(s1, s2)].append(background_distances.loc[s1, s2])
Z_scores = inter_cluster_distances.copy()
for s1 in Z_scores.index:
for s2 in Z_scores.columns:
mean = np.mean(background[(s1, s2)])
sd = np.std(background[(s1, s2)])
Z_scores.loc[s1, s2] = (inter_cluster_distances.loc[s1, s2] -
mean) / sd
return Z_scores