def grit(
similarity_melted_df: pd.DataFrame,
control_perts: List[str],
profile_col: str,
replicate_group_col: str,
replicate_summary_method: str = "mean",
) -> pd.DataFrame:
r"""Calculate grit
Parameters
----------
similarity_melted_df : pandas.DataFrame
a long pandas dataframe output from cytominer_eval.transform.metric_melt
control_perts : list
a list of control perturbations to calculate a null distribution
profile_col : str
the metadata column storing profile ids. The column can have unique or replicate
identifiers.
replicate_group_col : str
the metadata column indicating a higher order structure (group) than the
profile column. E.g. target gene vs. guide in a CRISPR experiment.
replicate_summary_method : {'mean', 'median'}, optional
how replicate z-scores to control perts are summarized. Defaults to "mean".
Returns
-------
pandas.DataFrame
A dataframe of grit measurements per perturbation
"""
# Check if we support the provided summary method
check_replicate_summary_method(replicate_summary_method)
# Determine pairwise replicates
similarity_melted_df = assign_replicates(
similarity_melted_df=similarity_melted_df,
replicate_groups=[profile_col, replicate_group_col],
)
# Check to make sure that the melted dataframe is full
assert_melt(similarity_melted_df, eval_metric="grit")
# Extract out specific columns
pair_ids = set_pair_ids()
profile_col_name = "{x}{suf}".format(
x=profile_col, suf=pair_ids[list(pair_ids)[0]]["suffix"])
# Define the columns to use in the calculation
column_id_info = set_grit_column_info(
profile_col=profile_col, replicate_group_col=replicate_group_col)
# Calculate grit for each perturbation
grit_df = (similarity_melted_df.groupby(profile_col_name).apply(
lambda x: calculate_grit(
replicate_group_df=x,
control_perts=control_perts,
column_id_info=column_id_info,
replicate_summary_method=replicate_summary_method,
)).reset_index(drop=True))
return grit_df
def test_set_pair_ids():
pair_a = "pair_a"
pair_b = "pair_b"
result = set_pair_ids()
assert result[pair_a]["index"] == "{pair_a}_index".format(pair_a=pair_a)
assert result[pair_a]["index"] == "{pair_a}_index".format(pair_a=pair_a)
assert result[pair_b]["suffix"] == "_{pair_b}".format(pair_b=pair_b)
assert result[pair_b]["suffix"] == "_{pair_b}".format(pair_b=pair_b)
def precision_recall(
similarity_melted_df: pd.DataFrame,
replicate_groups: List[str],
k: Union[int, List[int]],
) -> pd.DataFrame:
"""Determine the precision and recall at k for all unique replicate groups
based on a predefined similarity metric (see cytominer_eval.transform.metric_melt)
Parameters
----------
similarity_melted_df : pandas.DataFrame
An elongated symmetrical matrix indicating pairwise correlations between
samples. Importantly, it must follow the exact structure as output from
:py:func:`cytominer_eval.transform.transform.metric_melt`.
replicate_groups : List
a list of metadata column names in the original profile dataframe to use as
replicate columns.
k : List of ints or int
an integer indicating how many pairwise comparisons to threshold.
Returns
-------
pandas.DataFrame
precision and recall metrics for all replicate groups given k
"""
# Determine pairwise replicates and make sure to sort based on the metric!
similarity_melted_df = assign_replicates(
similarity_melted_df=similarity_melted_df, replicate_groups=replicate_groups
).sort_values(by="similarity_metric", ascending=False)
# Check to make sure that the melted dataframe is full
assert_melt(similarity_melted_df, eval_metric="precision_recall")
# Extract out specific columns
pair_ids = set_pair_ids()
replicate_group_cols = [
"{x}{suf}".format(x=x, suf=pair_ids[list(pair_ids)[0]]["suffix"])
for x in replicate_groups
]
# iterate over all k
precision_recall_df = pd.DataFrame()
if type(k) == int:
k = [k]
for k_ in k:
# Calculate precision and recall for all groups
precision_recall_df_at_k = similarity_melted_df.groupby(
replicate_group_cols
).apply(lambda x: calculate_precision_recall(x, k=k_))
precision_recall_df = precision_recall_df.append(precision_recall_df_at_k)
# Rename the columns back to the replicate groups provided
rename_cols = dict(zip(replicate_group_cols, replicate_groups))
return precision_recall_df.reset_index().rename(rename_cols, axis="columns")
def test_calculate_precision_recall():
similarity_melted_df = metric_melt(
df=df,
features=features,
metadata_features=meta_features,
similarity_metric="pearson",
eval_metric="precision_recall",
)
replicate_groups = ["Metadata_broad_sample"]
result = assign_replicates(similarity_melted_df=similarity_melted_df,
replicate_groups=replicate_groups).sort_values(
by="similarity_metric", ascending=False)
pair_ids = set_pair_ids()
replicate_group_cols = [
"{x}{suf}".format(x=x, suf=pair_ids[list(pair_ids)[0]]["suffix"])
for x in replicate_groups
]
example_group = result.groupby(replicate_group_cols).get_group(
name=("BRD-A38592941-001-02-7"))
assert example_group.shape[
0] == 383 * 6 # number of pairwise comparisons per dose
# Assert that the similarity metrics are sorted
assert (example_group.similarity_metric.diff().dropna() > 0).sum() == 0
# Perform the calculation!
result = pd.DataFrame(calculate_precision_recall(example_group, k=10),
columns=["result"])
expected_result = {"k": 10, "precision": 0.4, "recall": 0.1333}
expected_result = pd.DataFrame(expected_result,
index=["result"]).transpose()
assert_frame_equal(result, expected_result, check_less_precise=True)
# Check that recall is 1 when k is maximized
result = pd.DataFrame(
calculate_precision_recall(example_group, k=example_group.shape[0]),
columns=["result"],
)
assert result.loc["recall", "result"] == 1
def process_melt(
df: pd.DataFrame,
meta_df: pd.DataFrame,
eval_metric: str = "replicate_reproducibility",
) -> pd.DataFrame:
"""Helper function to annotate and process an input similarity matrix
Parameters
----------
df : pandas.DataFrame
A similarity matrix output from
:py:func:`cytominer_eval.transform.transform.get_pairwise_metric`
meta_df : pandas.DataFrame
A wide matrix of metadata information where the index aligns to the similarity
matrix index
eval_metric : str, optional
Which metric to ultimately calculate. Determines whether or not to keep the full
similarity matrix or only one diagonal. Defaults to "replicate_reproducibility".
Returns
-------
pandas.DataFrame
A pairwise similarity matrix
"""
# Confirm that the user formed the input arguments properly
assert df.shape[0] == df.shape[1], "Matrix must be symmetrical"
check_eval_metric(eval_metric)
# Get identifiers for pairing metadata
pair_ids = set_pair_ids()
# Subset the pairwise similarity metric depending on the eval metric given:
# "replicate_reproducibility" - requires only the upper triangle of a symmetric matrix
# "precision_recall" - requires the full symmetric matrix (no diagonal)
# Remove pairwise matrix diagonal and redundant pairwise comparisons
if eval_metric == "replicate_reproducibility":
upper_tri = get_upper_matrix(df)
df = df.where(upper_tri)
else:
np.fill_diagonal(df.values, np.nan)
# Convert pairwise matrix to melted (long) version based on index value
metric_unlabeled_df = (
pd.melt(
df.reset_index(),
id_vars="index",
value_vars=df.columns,
var_name=pair_ids["pair_b"]["index"],
value_name="similarity_metric",
)
.dropna()
.reset_index(drop=True)
.rename({"index": pair_ids["pair_a"]["index"]}, axis="columns")
)
# Merge metadata on index for both comparison pairs
output_df = meta_df.merge(
meta_df.merge(
metric_unlabeled_df,
left_index=True,
right_on=pair_ids["pair_b"]["index"],
),
left_index=True,
right_on=pair_ids["pair_a"]["index"],
suffixes=[pair_ids["pair_a"]["suffix"], pair_ids["pair_b"]["suffix"]],
).reset_index(drop=True)
return output_df
def assign_replicates(
similarity_melted_df: pd.DataFrame,
replicate_groups: List[str],
) -> pd.DataFrame:
"""Determine which profiles should be considered replicates.
Given an elongated pairwise correlation matrix with metadata annotations, determine
how to assign replicate information.
Parameters
----------
similarity_melted_df : pandas.DataFrame
Long pandas DataFrame of annotated pairwise correlations output from
:py:func:`cytominer_eval.transform.transform.metric_melt`.
replicate_groups : list
a list of metadata column names in the original profile dataframe used to
indicate replicate profiles.
Returns
-------
pd.DataFrame
A similarity_melted_df but with added columns indicating whether or not the
pairwise similarity metric is comparing replicates or not. Used in most eval
operations.
"""
pair_ids = set_pair_ids()
replicate_col_names = {
x: "{x}_replicate".format(x=x)
for x in replicate_groups
}
compare_dfs = []
for replicate_col in replicate_groups:
replicate_cols_with_suffix = [
"{col}{suf}".format(col=replicate_col, suf=pair_ids[x]["suffix"])
for x in pair_ids
]
assert all([
x in similarity_melted_df.columns
for x in replicate_cols_with_suffix
]), "replicate_group not found in melted dataframe columns"
replicate_col_name = replicate_col_names[replicate_col]
compare_df = similarity_melted_df.loc[:, replicate_cols_with_suffix]
compare_df.loc[:, replicate_col_name] = False
compare_df.loc[np.where(compare_df.iloc[:,
0] == compare_df.iloc[:,
1])[0],
replicate_col_name, ] = True
compare_dfs.append(compare_df)
compare_df = pd.concat(compare_dfs, axis="columns").reset_index(drop=True)
compare_df = compare_df.assign(
group_replicate=compare_df.loc[:, replicate_col_names.values()].min(
axis="columns")).loc[:,
list(replicate_col_names.values()) +
["group_replicate"]]
similarity_melted_df = similarity_melted_df.merge(compare_df,
left_index=True,
right_index=True)
return similarity_melted_df
]
features = df.drop(meta_features, axis="columns").columns.tolist()
similarity_melted_df = metric_melt(
df=df,
features=features,
metadata_features=meta_features,
similarity_metric="pearson",
eval_metric="grit",
)
control_perts = ["Luc-2", "LacZ-2", "LacZ-3"]
profile_col = "Metadata_pert_name"
replicate_group_col = "Metadata_gene_name"
pair_ids = set_pair_ids()
replicate_col_name = "{x}{suf}".format(
x=profile_col, suf=pair_ids[list(pair_ids)[0]]["suffix"])
column_id_info = set_grit_column_info(profile_col=profile_col,
replicate_group_col=replicate_group_col)
def test_get_grit_entry():
with pytest.raises(AssertionError) as ae:
result = get_grit_entry(df=similarity_melted_df,
col=replicate_col_name)
assert "grit is calculated for each perturbation independently" in str(
ae.value)
expected_result = "EMPTY"
def precision_recall(
similarity_melted_df: pd.DataFrame,
replicate_groups: List[str],
groupby_columns: List[str],
k: Union[int, List[int], str],
) -> pd.DataFrame:
"""Determine the precision and recall at k for all unique groupby_columns samples
based on a predefined similarity metric (see cytominer_eval.transform.metric_melt)
Parameters
----------
similarity_melted_df : pandas.DataFrame
An elongated symmetrical matrix indicating pairwise correlations between
samples. Importantly, it must follow the exact structure as output from
:py:func:`cytominer_eval.transform.transform.metric_melt`.
replicate_groups : List
a list of metadata column names in the original profile dataframe to use as replicate columns.
groupby_columns : List of str
Column by which the similarity matrix is grouped and by which the precision/recall is calculated.
For example, if groupby_column = Metadata_sample then the precision is calculated for each sample.
Calculating the precision by sample is the default
but it is mathematically not incorrect to calculate the precision at the MOA level.
This is just less intuitive to understand.
k : List of ints or int
an integer indicating how many pairwise comparisons to threshold.
if k = 'R' then precision at R will be calculated where R is the number of other replicates
Returns
-------
pandas.DataFrame
precision and recall metrics for all groupby_column groups given k
"""
# Check for correct k input
assert Union[int, List[int], str]
# Determine pairwise replicates and make sure to sort based on the metric!
similarity_melted_df = assign_replicates(
similarity_melted_df=similarity_melted_df,
replicate_groups=replicate_groups).sort_values(by="similarity_metric",
ascending=False)
# Check to make sure that the melted dataframe is full
assert_melt(similarity_melted_df, eval_metric="precision_recall")
# Extract out specific columns
pair_ids = set_pair_ids()
groupby_cols_suffix = [
"{x}{suf}".format(x=x, suf=pair_ids[list(pair_ids)[0]]["suffix"])
for x in groupby_columns
]
# iterate over all k
precision_recall_df = pd.DataFrame()
if type(k) == int:
k = [k]
for k_ in k:
# Calculate precision and recall for all groups
precision_recall_df_at_k = similarity_melted_df.groupby(
groupby_cols_suffix).apply(
lambda x: calculate_precision_recall(x, k=k_))
precision_recall_df = precision_recall_df.append(
precision_recall_df_at_k)
# Rename the columns back to the replicate groups provided
rename_cols = dict(zip(groupby_cols_suffix, groupby_columns))
return precision_recall_df.reset_index().rename(rename_cols,
axis="columns")
def hitk(
similarity_melted_df: pd.DataFrame,
replicate_groups: List[str],
groupby_columns: List[str],
percent_list: Union[int, List[int]],
) -> pd.DataFrame:
"""Calculate the hit@k hits list and percent scores.
This function groups the similarity matrix by each sample (group_col) and by similarity score. It then determines the rank of each correct hit.
A correct hit is a connection to another sample with the same replicate attributes (replicate_groups), for example the same MOA.
Hit@k records all hits/indexes in a long list which can be used to create histogram plots or similar visualizations.
The percent scores contain the number of hits above the expected random distribution at a given percentage.
For example, at 5 percent we calculate how many hits are within the first 5 percent of classes (number of neighbors) and then subtract the expected number of hits.
Parameters
----------
similarity_melted_df : pandas.DataFrame
An elongated symmetrical matrix indicating pairwise correlations between
samples. Importantly, it must follow the exact structure as output from
:py:func:`cytominer_eval.transform.transform.metric_melt`.
replicate_groups : list or int
a list of metadata column names in the original profile dataframe to use as replicate columns.
groupby_columns: str
group columns determine the columns over which the similarity_melted_df is grouped.
Usually groupby_columns will span the full space of the input data
such that drop_duplicates by the groupby_cols would not change the data.
If you group over Metadata_plate for examples, you will get meaningless results.
This can easily be seen from the fact that the percent score at 100 will be nonzero.
percent_list : list or "all"
A list of percentages at which to calculate the percent scores, ie the amount of hits below this percentage.
If percent_list == "all" a full dict with the length of classes will be created.
Percentages are given as integers, ie 50 means 50 %.
Returns
-------
hits_list : list
full list of all hits. Can be used for histogram plotting.
percent_scores: dict
dictionary of the percentage list and their corresponding percent scores (see percent score function).
"""
# make sure percent_list is a list
if type(percent_list) == int:
percent_list = [percent_list]
# check for correct input
assert type(percent_list) == list or percent_list == "all", "input is incorrect"
if type(percent_list) == list:
assert max(percent_list) <= 100, "percentages must be smaller than 100"
similarity_melted_df = assign_replicates(
similarity_melted_df=similarity_melted_df, replicate_groups=replicate_groups
)
# Check to make sure that the melted dataframe is full
assert_melt(similarity_melted_df, eval_metric="hitk")
# Extract the name of the columns in the sim_df
pair_ids = set_pair_ids()
groupby_cols_suffix = [
"{x}{suf}".format(x=x, suf=pair_ids[list(pair_ids)[0]]["suffix"])
for x in groupby_columns
]
# group the sim_df by the groupby_columns
grouped = similarity_melted_df.groupby(groupby_cols_suffix)
nr_of_groups = grouped.ngroups
# Within each group, add the ranks of each connection to a new column
similarity_melted_with_rank = grouped.apply(lambda x: add_hit_rank(x))
# make a list of the ranks of correct connection (hits), ie where the group_replicate is true
hits_list = similarity_melted_with_rank[
similarity_melted_with_rank["group_replicate"] == True
]["rank"].tolist()
# calculate the scores at each percentage
percent_scores = percentage_scores(hits_list, percent_list, nr_of_groups)
return hits_list, percent_scores