def _pick_candidates(
*,
adata_of_all_genes_of_all_cells: AnnData,
what: Union[str, ut.Matrix] = "__x__",
max_gene_cell_fraction: float,
min_gene_maximum: int,
min_genes_of_modules: int,
allowed_genes_mask: ut.NumpyVector,
) -> Optional[Tuple[AnnData, ut.NumpyVector]]:
data = ut.get_vo_proper(adata_of_all_genes_of_all_cells,
what,
layout="column_major")
nnz_cells_of_genes = ut.nnz_per(data, per="column")
nnz_cell_fraction_of_genes = nnz_cells_of_genes / adata_of_all_genes_of_all_cells.n_obs
nnz_cell_fraction_mask_of_genes = nnz_cell_fraction_of_genes <= max_gene_cell_fraction
max_umis_of_genes = ut.max_per(data, per="column")
max_umis_mask_of_genes = max_umis_of_genes >= min_gene_maximum
candidates_mask_of_genes = max_umis_mask_of_genes & nnz_cell_fraction_mask_of_genes & allowed_genes_mask
ut.log_calc("candidate_genes", candidates_mask_of_genes)
candidate_genes_indices = np.where(candidates_mask_of_genes)[0]
candidate_genes_count = candidate_genes_indices.size
if candidate_genes_count < min_genes_of_modules:
return None
candidate_data = ut.slice(adata_of_all_genes_of_all_cells,
name=".candidate_genes",
vars=candidate_genes_indices,
top_level=False)
return candidate_data, candidate_genes_indices
def _filter_genes(
*,
cells_count: int,
genes_count: int,
fold_factors: ut.CompressedMatrix,
min_gene_fold_factor: float,
max_gene_fraction: Optional[float] = None,
) -> ut.NumpyVector:
ut.timed_parameters(cells=cells_count, genes=genes_count, fold_factors=fold_factors.nnz)
max_fold_factors_of_genes = ut.max_per(fold_factors, per="column")
assert max_fold_factors_of_genes.size == genes_count
mask_of_deviant_genes = max_fold_factors_of_genes >= min_gene_fold_factor
deviant_gene_fraction = np.sum(mask_of_deviant_genes) / genes_count
if max_gene_fraction is not None and deviant_gene_fraction > max_gene_fraction:
if ut.logging_calc():
ut.log_calc("candidate_deviant_genes", mask_of_deviant_genes)
quantile_gene_fold_factor = np.quantile(max_fold_factors_of_genes, 1 - max_gene_fraction)
assert quantile_gene_fold_factor is not None
ut.log_calc("quantile_gene_fold_factor", quantile_gene_fold_factor)
if quantile_gene_fold_factor > min_gene_fold_factor:
min_gene_fold_factor = quantile_gene_fold_factor
mask_of_deviant_genes = max_fold_factors_of_genes >= min_gene_fold_factor
fold_factors.data[fold_factors.data < min_gene_fold_factor] = 0
ut.eliminate_zeros(fold_factors)
if ut.logging_calc():
ut.log_calc("deviant_genes", mask_of_deviant_genes)
deviant_gene_indices = np.where(mask_of_deviant_genes)[0]
return deviant_gene_indices
def _compute_metacell_inner_folds(
*,
metacell_index: int,
cells_data: ut.Matrix,
metacells_data: ut.Matrix,
group_of_cells: ut.NumpyVector,
total_umis_per_cell: ut.NumpyVector,
total_umis_per_metacell: ut.NumpyVector,
) -> ut.NumpyVector:
grouped_cells_mask = group_of_cells == metacell_index
assert np.sum(grouped_cells_mask) > 0
grouped_cells_data = ut.to_numpy_matrix(cells_data[grouped_cells_mask, :])
total_umis_per_grouped_cell = total_umis_per_cell[grouped_cells_mask]
metacell_data = metacells_data[metacell_index, :]
total_umis_of_metacell = total_umis_per_metacell[metacell_index]
expected_scale_per_grouped_cell = total_umis_per_grouped_cell / total_umis_of_metacell
expected_cells_data = expected_scale_per_grouped_cell[:, np.
newaxis] * metacell_data[
np.newaxis, :]
assert expected_cells_data.shape == grouped_cells_data.shape
fold_factors_per_grouped_cell = np.log2(
(expected_cells_data + 1) / (grouped_cells_data + 1))
fold_factors = ut.max_per(ut.to_layout(fold_factors_per_grouped_cell,
"column_major"),
per="column")
max_fold_factor = np.max(fold_factors)
ut.log_calc("max_fold_factor", max_fold_factor)
return fold_factors
def _test_per(rows_matrix: ut.Matrix) -> None:
columns_matrix = ut.to_layout(rows_matrix, layout="column_major")
assert np.allclose(ut.nnz_per(rows_matrix, per="row"), np.array([2, 3]))
assert np.allclose(ut.nnz_per(columns_matrix, per="column"), np.array([1, 2, 2]))
assert np.allclose(ut.sum_per(rows_matrix, per="row"), np.array([3, 12]))
assert np.allclose(ut.sum_per(columns_matrix, per="column"), np.array([3, 5, 7]))
assert np.allclose(ut.max_per(rows_matrix, per="row"), np.array([2, 5]))
assert np.allclose(ut.max_per(columns_matrix, per="column"), np.array([3, 4, 5]))
assert np.allclose(ut.min_per(rows_matrix, per="row"), np.array([0, 3]))
assert np.allclose(ut.min_per(columns_matrix, per="column"), np.array([0, 1, 2]))
assert np.allclose(ut.sum_squared_per(rows_matrix, per="row"), np.array([5, 50]))
assert np.allclose(ut.sum_squared_per(columns_matrix, per="column"), np.array([9, 17, 29]))
assert np.allclose(ut.fraction_per(rows_matrix, per="row"), np.array([3 / 15, 12 / 15]))
assert np.allclose(ut.fraction_per(columns_matrix, per="column"), np.array([3 / 15, 5 / 15, 7 / 15]))
assert np.allclose(ut.mean_per(rows_matrix, per="row"), np.array([3 / 3, 12 / 3]))
assert np.allclose(ut.mean_per(columns_matrix, per="column"), np.array([3 / 2, 5 / 2, 7 / 2]))
assert np.allclose(
ut.variance_per(rows_matrix, per="row"), np.array([5 / 3 - (3 / 3) ** 2, 50 / 3 - (12 / 3) ** 2])
)
assert np.allclose(
ut.variance_per(columns_matrix, per="column"),
np.array([9 / 2 - (3 / 2) ** 2, 17 / 2 - (5 / 2) ** 2, 29 / 2 - (7 / 2) ** 2]),
)
assert np.allclose(
ut.normalized_variance_per(columns_matrix, per="column"),
np.array(
[(9 / 2 - (3 / 2) ** 2) / (3 / 2), (17 / 2 - (5 / 2) ** 2) / (5 / 2), (29 / 2 - (7 / 2) ** 2) / (7 / 2)]
),
)
dense = ut.to_numpy_matrix(ut.fraction_by(rows_matrix, by="row"))
assert np.allclose(dense, np.array([[0 / 3, 1 / 3, 2 / 3], [3 / 12, 4 / 12, 5 / 12]]))
dense = ut.to_numpy_matrix(ut.fraction_by(columns_matrix, by="column"))
assert np.allclose(dense, np.array([[0 / 3, 1 / 5, 2 / 7], [3 / 3, 4 / 5, 5 / 7]]))
def significant_folds_matrix(
folds: ut.NumpyMatrix,
min_significant_value: float,
min_entry_value: float,
abs_folds: bool,
) -> ut.CompressedMatrix:
"""
Convert a dense folds matrix to a sparse one containing just the significant values.
"""
folds_by_column = ut.to_layout(folds, layout="column_major")
if abs_folds:
comparable_folds_by_column = np.abs(folds_by_column)
else:
comparable_folds_by_column = folds_by_column
max_fold_per_gene = ut.max_per(comparable_folds_by_column, per="column")
significant_genes_mask = max_fold_per_gene >= min_significant_value
ut.log_calc("significant_genes_mask", significant_genes_mask)
folds_by_column[:, ~significant_genes_mask] = 0
folds_by_column[comparable_folds_by_column < min_entry_value] = 0
folds_by_row = ut.to_layout(folds_by_column, layout="row_major")
return sparse.csr_matrix(folds_by_row)
def find_noisy_lonely_genes( # pylint: disable=too-many-statements
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
excluded_genes_mask: Optional[str] = None,
max_sampled_cells: int = pr.noisy_lonely_max_sampled_cells,
downsample_min_samples: int = pr.noisy_lonely_downsample_min_samples,
downsample_min_cell_quantile: float = pr.
noisy_lonely_downsample_max_cell_quantile,
downsample_max_cell_quantile: float = pr.
noisy_lonely_downsample_min_cell_quantile,
min_gene_total: int = pr.noisy_lonely_min_gene_total,
min_gene_normalized_variance: float = pr.
noisy_lonely_min_gene_normalized_variance,
max_gene_similarity: float = pr.noisy_lonely_max_gene_similarity,
random_seed: int = pr.random_seed,
inplace: bool = True,
) -> Optional[ut.PandasSeries]:
"""
Detect "noisy lonely" genes based on ``what`` (default: {what}) data.
Return the indices of genes which are "noisy" (have high variance compared to their mean) and
also "lonely" (have low correlation with all other genes). Such genes should be excluded since
they will never meaningfully help us compute groups, and will actively cause profiles to be
considered "deviants".
Noisy genes have high expression and variance. Lonely genes have no (or low) correlations with
any other gene. Noisy lonely genes tend to throw off clustering algorithms. In general, such
algorithms try to group together cells with the same overall biological state. Since the genes
are lonely, they don't contribute towards this goal. Since they are noisy, they actively hamper
this, because they make cells which are otherwise similar appear different (just for this lonely
gene).
It is therefore useful to explicitly identify, in a pre-processing step, the (few) such genes,
and exclude them from the rest of the analysis.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Variable (Gene) Annotations
``noisy_lonely_genes``
A boolean mask indicating whether each gene was found to be a "noisy lonely" gene.
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas series (indexed by the variable names).
**Computation Parameters**
1. If we have more than ``max_sampled_cells`` (default: {max_sampled_cells}), pick this number
of random cells from the data using the ``random_seed``.
2. If we were specified an ``excluded_genes_mask``, this is the name of a per-variable (gene)
annotation containing a mask of excluded genes. Get rid of all these excluded genes.
3. Invoke :py:func:`metacells.tools.downsample.downsample_cells` to downsample the cells to the
same total number of UMIs, using the ``downsample_min_samples`` (default:
{downsample_min_samples}), ``downsample_min_cell_quantile`` (default:
{downsample_min_cell_quantile}), ``downsample_max_cell_quantile`` (default:
{downsample_max_cell_quantile}) and the ``random_seed`` (default: {random_seed}).
4. Find "noisy" genes which have a total number of UMIs of at least ``min_gene_total`` (default:
{min_gene_total}) and a normalized variance of at least ``min_gene_normalized_variance``
(default: ``min_gene_normalized_variance``).
5. Cross-correlate the noisy genes.
6. Find the noisy "lonely" genes whose maximal correlation is at most
``max_gene_similarity`` (default: {max_gene_similarity}) with all other genes.
"""
if max_sampled_cells < adata.n_obs:
np.random.seed(random_seed)
cell_indices = np.random.choice(np.arange(adata.n_obs),
size=max_sampled_cells,
replace=False)
s_data = ut.slice(adata,
obs=cell_indices,
name=".sampled",
top_level=False)
else:
s_data = ut.copy_adata(adata, top_level=False)
track_var: Optional[str] = "sampled_gene_index"
if excluded_genes_mask is not None:
results = filter_data(s_data,
name="included",
top_level=False,
track_var=track_var,
var_masks=[f"~{excluded_genes_mask}"])
track_var = None
assert results is not None
i_data = results[0]
assert i_data is not None
else:
i_data = s_data
downsample_cells(
i_data,
what,
downsample_min_samples=downsample_min_samples,
downsample_min_cell_quantile=downsample_min_cell_quantile,
downsample_max_cell_quantile=downsample_max_cell_quantile,
random_seed=random_seed,
)
find_high_total_genes(i_data, "downsampled", min_gene_total=min_gene_total)
results = filter_data(i_data,
name="high_total",
top_level=False,
track_var=track_var,
var_masks=["high_total_gene"])
track_var = None
assert results is not None
ht_data = results[0]
noisy_lonely_genes_mask = np.full(adata.n_vars, False)
if ht_data is not None:
ht_genes_count = ht_data.shape[1]
ht_gene_ht_gene_similarity_frame = compute_var_var_similarity(
ht_data,
"downsampled",
inplace=False,
reproducible=(random_seed != 0))
assert ht_gene_ht_gene_similarity_frame is not None
ht_gene_ht_gene_similarity_matrix = ut.to_numpy_matrix(
ht_gene_ht_gene_similarity_frame, only_extract=True)
ht_gene_ht_gene_similarity_matrix = ut.to_layout(
ht_gene_ht_gene_similarity_matrix,
layout="row_major",
symmetric=True)
np.fill_diagonal(ht_gene_ht_gene_similarity_matrix, -1)
htv_mask_series = find_high_normalized_variance_genes(
ht_data,
"downsampled",
min_gene_normalized_variance=min_gene_normalized_variance,
inplace=False)
assert htv_mask_series is not None
htv_mask = ut.to_numpy_vector(htv_mask_series)
htv_genes_count = np.sum(htv_mask)
assert htv_genes_count <= ht_genes_count
if htv_genes_count > 0:
htv_gene_ht_gene_similarity_matrix = ht_gene_ht_gene_similarity_matrix[
htv_mask, :]
assert ut.is_layout(htv_gene_ht_gene_similarity_matrix,
"row_major")
assert htv_gene_ht_gene_similarity_matrix.shape == (
htv_genes_count, ht_genes_count)
max_similarity_of_htv_genes = ut.max_per(
htv_gene_ht_gene_similarity_matrix, per="row")
htvl_mask = max_similarity_of_htv_genes <= max_gene_similarity
htvl_genes_count = np.sum(htvl_mask)
ut.log_calc("noisy_lonely_genes_count", htvl_genes_count)
if htvl_genes_count > 0:
base_index_of_ht_genes = ut.get_v_numpy(
ht_data, "sampled_gene_index")
base_index_of_htv_genes = base_index_of_ht_genes[htv_mask]
base_index_of_htvl_genes = base_index_of_htv_genes[htvl_mask]
noisy_lonely_genes_mask[base_index_of_htvl_genes] = True
htvl_gene_ht_gene_similarity_matrix = htv_gene_ht_gene_similarity_matrix[
htvl_mask, :]
htvl_gene_ht_gene_similarity_matrix = ut.to_layout(
htvl_gene_ht_gene_similarity_matrix, layout="row_major")
assert htvl_gene_ht_gene_similarity_matrix.shape == (
htvl_genes_count, ht_genes_count)
if ut.logging_calc():
i_gene_totals = ut.get_v_numpy(i_data,
"downsampled",
sum=True)
ht_mask = ut.get_v_numpy(i_data, "high_total_gene")
i_total = np.sum(i_gene_totals)
htvl_gene_totals = i_gene_totals[ht_mask][htv_mask][
htvl_mask]
top_similarity_of_htvl_genes = ut.top_per(
htvl_gene_ht_gene_similarity_matrix, 10, per="row")
for htvl_index, gene_index in enumerate(
base_index_of_htvl_genes):
gene_name = adata.var_names[gene_index]
gene_total = htvl_gene_totals[htvl_index]
gene_percent = 100 * gene_total / i_total
similar_ht_values = ut.to_numpy_vector(
top_similarity_of_htvl_genes[htvl_index, :]) #
assert len(similar_ht_values) == ht_genes_count
top_similar_ht_mask = similar_ht_values > 0
top_similar_ht_values = similar_ht_values[
top_similar_ht_mask]
top_similar_ht_indices = base_index_of_ht_genes[
top_similar_ht_mask]
top_similar_ht_names = adata.var_names[
top_similar_ht_indices]
ut.log_calc(
f"- {gene_name}",
f"total downsampled UMIs: {gene_total} " +
f"({gene_percent:.4g}%), correlated with: " +
", ".join([
f"{similar_gene_name}: {similar_gene_value:.4g}"
for similar_gene_value, similar_gene_name in
reversed(
sorted(
zip(top_similar_ht_values,
top_similar_ht_names)))
]),
)
if ut.logging_calc():
ut.log_calc("noisy_lonely_gene_names",
sorted(list(adata.var_names[noisy_lonely_genes_mask])))
if inplace:
ut.set_v_data(adata, "noisy_lonely_gene", noisy_lonely_genes_mask)
return None
ut.log_return("noisy_lonely_genes", noisy_lonely_genes_mask)
return ut.to_pandas_series(noisy_lonely_genes_mask, index=adata.var_names)
def _related_genes( # pylint: disable=too-many-statements,too-many-branches
*,
adata_of_all_genes_of_all_cells: AnnData,
what: Union[str, ut.Matrix] = "__x__",
rare_gene_indices_of_modules: List[List[int]],
allowed_genes_mask: ut.NumpyVector,
min_genes_of_modules: int,
min_gene_maximum: int,
min_cells_of_modules: int,
max_cells_of_modules: int,
min_cell_module_total: int,
min_related_gene_fold_factor: float,
max_related_gene_increase_factor: float,
) -> List[List[int]]:
total_all_cells_umis_of_all_genes = ut.get_v_numpy(
adata_of_all_genes_of_all_cells, what, sum=True)
ut.log_calc("genes for modules:")
modules_count = 0
related_gene_indices_of_modules: List[List[int]] = []
rare_gene_indices_of_any: Set[int] = set()
for rare_gene_indices_of_module in rare_gene_indices_of_modules:
if len(rare_gene_indices_of_module) >= min_genes_of_modules:
rare_gene_indices_of_any.update(list(rare_gene_indices_of_module))
for rare_gene_indices_of_module in rare_gene_indices_of_modules:
if len(rare_gene_indices_of_module) < min_genes_of_modules:
continue
module_index = modules_count
modules_count += 1
with ut.log_step("- module", module_index):
ut.log_calc(
"rare_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[rare_gene_indices_of_module]))
adata_of_module_genes_of_all_cells = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.rare_gene",
vars=rare_gene_indices_of_module,
top_level=False,
)
total_module_genes_umis_of_all_cells = ut.get_o_numpy(
adata_of_module_genes_of_all_cells, what, sum=True)
mask_of_expressed_cells = total_module_genes_umis_of_all_cells > 0
expressed_cells_count = np.sum(mask_of_expressed_cells)
if expressed_cells_count > max_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"expressed_cells",
ut.mask_description(mask_of_expressed_cells) +
" (too many)")
continue
if expressed_cells_count < min_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"expressed_cells",
ut.mask_description(mask_of_expressed_cells) +
" (too few)")
continue
ut.log_calc("expressed_cells", mask_of_expressed_cells)
adata_of_all_genes_of_expressed_cells_of_module = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.rare_cell",
obs=mask_of_expressed_cells,
top_level=False,
)
total_expressed_cells_umis_of_all_genes = ut.get_v_numpy(
adata_of_all_genes_of_expressed_cells_of_module,
what,
sum=True)
data = ut.get_vo_proper(
adata_of_all_genes_of_expressed_cells_of_module,
what,
layout="column_major")
max_expressed_cells_umis_of_all_genes = ut.max_per(data,
per="column")
total_background_cells_umis_of_all_genes = (
total_all_cells_umis_of_all_genes -
total_expressed_cells_umis_of_all_genes)
expressed_cells_fraction_of_all_genes = total_expressed_cells_umis_of_all_genes / sum(
total_expressed_cells_umis_of_all_genes)
background_cells_fraction_of_all_genes = total_background_cells_umis_of_all_genes / sum(
total_background_cells_umis_of_all_genes)
mask_of_related_genes = (
allowed_genes_mask
& (max_expressed_cells_umis_of_all_genes >= min_gene_maximum)
& (expressed_cells_fraction_of_all_genes >=
background_cells_fraction_of_all_genes *
(2**min_related_gene_fold_factor)))
related_gene_indices = np.where(mask_of_related_genes)[0]
assert np.all(mask_of_related_genes[rare_gene_indices_of_module])
base_genes_of_all_cells_adata = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.base",
vars=rare_gene_indices_of_module)
total_base_genes_of_all_cells = ut.get_o_numpy(
base_genes_of_all_cells_adata, what, sum=True)
mask_of_strong_base_cells = total_base_genes_of_all_cells >= min_cell_module_total
count_of_strong_base_cells = np.sum(mask_of_strong_base_cells)
if ut.logging_calc():
ut.log_calc(
"candidate_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[related_gene_indices]))
ut.log_calc("base_strong_genes", count_of_strong_base_cells)
related_gene_indices_of_module = list(rare_gene_indices_of_module)
for gene_index in related_gene_indices:
if gene_index in rare_gene_indices_of_module:
continue
if gene_index in rare_gene_indices_of_any:
ut.log_calc(
f"- candidate gene {adata_of_all_genes_of_all_cells.var_names[gene_index]} "
f"belongs to another module")
continue
if gene_index not in rare_gene_indices_of_module:
related_gene_of_all_cells_adata = ut.slice(
adata_of_all_genes_of_all_cells,
name=
f".{adata_of_all_genes_of_all_cells.var_names[gene_index]}",
vars=np.array([gene_index]),
)
assert related_gene_of_all_cells_adata.n_vars == 1
total_related_genes_of_all_cells = ut.get_o_numpy(
related_gene_of_all_cells_adata, what, sum=True)
total_related_genes_of_all_cells += total_base_genes_of_all_cells
mask_of_strong_related_cells = total_related_genes_of_all_cells >= min_cell_module_total
count_of_strong_related_cells = np.sum(
mask_of_strong_related_cells)
ut.log_calc(
f"- candidate gene {adata_of_all_genes_of_all_cells.var_names[gene_index]} "
f"strong cells: {count_of_strong_related_cells} "
f"factor: {count_of_strong_related_cells / count_of_strong_base_cells}"
)
if count_of_strong_related_cells > max_related_gene_increase_factor * count_of_strong_base_cells:
continue
related_gene_indices_of_module.append(gene_index)
related_gene_indices_of_modules.append(
related_gene_indices_of_module) #
if ut.logging_calc():
ut.log_calc("related genes for modules:")
for module_index, related_gene_indices_of_module in enumerate(
related_gene_indices_of_modules):
ut.log_calc(
f"- module {module_index} related_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[related_gene_indices_of_module]),
)
return related_gene_indices_of_modules
def compute_inner_fold_factors(
what: Union[str, ut.Matrix] = "__x__",
*,
adata: AnnData,
gdata: AnnData,
group: Union[str, ut.Vector] = "metacell",
min_gene_inner_fold_factor: float = pr.min_gene_inner_fold_factor,
min_entry_inner_fold_factor: float = pr.min_entry_inner_fold_factor,
inner_abs_folds: float = pr.inner_abs_folds,
) -> None:
"""
Compute the inner fold factors of genes within in each metacell.
This computes, for each cell of the metacell, the same fold factors that are used to detect deviant cells (see
:py:func:`metacells.tools.deviants.find_deviant_cells`), and keeps the maximal fold for each gene in the metacell.
The result per-metacell-per-gene matrix is then made sparse by discarding too-low values (setting them to zero).
Ideally, this matrix should be "very" sparse. If it contains "too many" non-zero values, this indicates the
metacells contains "too much" variability. This may be due to actual biology (e.g. immune cells or olfactory nerves
which are all similar except for each one expressing one different gene), due to batch effects (similar cells in
distinct batches differing in some genes due to technical issues), due to low data quality (the overall noise level
is so high that this is simply the best the algorithm can do), or worse - a combination of the above.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where ``what`` is a
per-variable-per-observation matrix or the name of a per-variable-per-observation annotation containing such a
matrix.
In addition, ``gdata`` is assumed to have one observation for each group, and use the same
genes as ``adata``.
**Returns**
Sets the following in ``gdata``:
Per-Variable Per-Observation (Gene-Cell) Annotations
``inner_fold``
For each gene and group, the maximal fold factor of this gene in any cell contained in the group (unless the
value is too low to be of interest, in which case it will be zero).
**Computation Parameters**
1. For each group (metacell), for each gene, compute the gene's maximal (in all the cells of the group) fold factor
log2((actual UMIs + 1) / (expected UMIs + 1)), similarly to
:py:func:`metacells.tools.deviants.find_deviant_cells`.
2. If the maximal fold factor for a gene (across all metacells) is below ``min_gene_inner_fold_factor`` (default:
{min_gene_inner_fold_factor}), then set all the gene's fold factors to zero (too low to be of interest). If
``inner_abs_folds`` (default: {inner_abs_folds}), consider the absolute fold factors.
3. Otherwise, for any metacell whose fold factor for the gene is less than ``min_entry_inner_fold_factor`` (default:
{min_entry_inner_fold_factor}), set the fold factor to zero (too low to be of interest).
"""
assert 0 <= min_entry_inner_fold_factor <= min_gene_inner_fold_factor
cells_data = ut.get_vo_proper(adata, what, layout="row_major")
metacells_data = ut.get_vo_proper(gdata, what, layout="row_major")
group_of_cells = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
total_umis_per_cell = ut.sum_per(cells_data, per="row")
total_umis_per_metacell = ut.sum_per(metacells_data, per="row")
@ut.timed_call("compute_metacell_inner_folds")
def _compute_single_metacell_inner_folds(
metacell_index: int) -> ut.NumpyVector:
return _compute_metacell_inner_folds(
metacell_index=metacell_index,
cells_data=cells_data,
metacells_data=metacells_data,
group_of_cells=group_of_cells,
total_umis_per_cell=total_umis_per_cell,
total_umis_per_metacell=total_umis_per_metacell,
)
results = list(
ut.parallel_map(_compute_single_metacell_inner_folds, gdata.n_obs))
dense_inner_folds_by_row = np.array(results)
dense_inner_folds_by_column = ut.to_layout(dense_inner_folds_by_row,
"column_major")
if inner_abs_folds:
comparable_dense_inner_folds_by_column = np.abs(
dense_inner_folds_by_column)
else:
comparable_dense_inner_folds_by_column = dense_inner_folds_by_column
max_fold_per_gene = ut.max_per(comparable_dense_inner_folds_by_column,
per="column")
significant_genes_mask = max_fold_per_gene >= min_gene_inner_fold_factor
ut.log_calc("significant_genes_mask", significant_genes_mask)
dense_inner_folds_by_column[:, ~significant_genes_mask] = 0
dense_inner_folds_by_column[comparable_dense_inner_folds_by_column <
min_entry_inner_fold_factor] = 0
dense_inner_folds_by_row = ut.to_layout(dense_inner_folds_by_column,
layout="row_major")
sparse_inner_folds = sparse.csr_matrix(dense_inner_folds_by_row)
ut.set_vo_data(gdata, "inner_fold", sparse_inner_folds)
def find_metacells_significant_genes(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
min_gene_range_fold: float = pr.min_significant_metacells_gene_range_fold_factor,
normalization: float = pr.metacells_gene_range_normalization,
min_gene_fraction: float = pr.min_significant_metacells_gene_fraction,
inplace: bool = True,
) -> Optional[ut.PandasSeries]:
"""
Find genes which have a significant signal in metacells data. This computation is too unreliable to be used on
cells.
Find genes which have a high maximal expression in at least one metacell, and a wide range of expression across the
metacells. Such genes are good candidates for being used as marker genes and/or to compute distances between
metacells.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Variable (Gene) Annotations
``significant_gene``
A boolean mask indicating whether each gene was found to be significant.
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas series (indexed by the variable names).
**Computation Parameters**
1. Compute the minimal and maximal expression level of each gene.
2. Select the genes whose fold factor (log2 of maximal over minimal value, using the ``normalization``
(default: {normalization}) is at least ``min_gene_range_fold`` (default: {min_gene_range_fold}).
3. Select the genes whose maximal expression is at least ``min_gene_fraction`` (default: {min_gene_fraction}).
"""
assert normalization >= 0
data = ut.get_vo_proper(adata, what, layout="row_major")
fractions_of_genes = ut.to_layout(ut.fraction_by(data, by="row"), layout="column_major")
min_fraction_of_genes = ut.min_per(fractions_of_genes, per="column")
max_fraction_of_genes = ut.max_per(fractions_of_genes, per="column")
high_max_fraction_genes_mask = max_fraction_of_genes >= min_gene_fraction
ut.log_calc("high max fraction genes", high_max_fraction_genes_mask)
min_fraction_of_genes += normalization
max_fraction_of_genes += normalization
max_fraction_of_genes /= min_fraction_of_genes
range_fold_of_genes = np.log2(max_fraction_of_genes, out=max_fraction_of_genes)
high_range_genes_mask = range_fold_of_genes >= min_gene_range_fold
ut.log_calc("high range genes", high_range_genes_mask)
significant_genes_mask = high_max_fraction_genes_mask & high_range_genes_mask
if inplace:
ut.set_v_data(adata, "significant_gene", significant_genes_mask)
return None
ut.log_return("significant_genes", significant_genes_mask)
return ut.to_pandas_series(significant_genes_mask, index=adata.var_names)