def spread_coordinates(
adata: AnnData,
*,
prefix: str = "umap",
suffix: str = "spread",
cover_fraction: float = pr.cover_fraction,
noise_fraction: float = pr.noise_fraction,
random_seed: int = pr.random_seed,
) -> None:
"""
Move UMAP points so they cover some fraction of the plot area without overlapping.
**Input**
The input annotated ``adata`` is expected to contain the per-observation properties
``<prefix>_x`` and ``<prefix>_y`` (default prefix: {prefix}) which contain the UMAP coordinates.
**Returns**
Sets the following annotations in ``adata``:
Observation (Cell) Annotations
``<prefix>_x_<suffix>``, ``<prefix>_y_<suffix>`` (default suffix: {suffix})
The new coordinates which will be spread out so the points do not overlap and
cover some fraction of the total plot area.
**Computation Parameters**
1. Move the points so they cover ``cover_fraction`` (default: {cover_fraction}) of the total
plot area. Also add a noise of the ``noise_fraction`` (default: {noise_fraction}) of the
minimal distance between the
points, using the ``random_seed`` (default: {random_seed}).
"""
assert 0 < cover_fraction < 1
assert noise_fraction >= 0
x_coordinates = ut.get_o_numpy(adata, f"{prefix}_x")
y_coordinates = ut.get_o_numpy(adata, f"{prefix}_y")
x_coordinates, y_coordinates = ut.cover_coordinates(
x_coordinates,
y_coordinates,
cover_fraction=cover_fraction,
noise_fraction=noise_fraction,
random_seed=random_seed,
)
ut.set_o_data(adata, f"{prefix}_x_{suffix}", x_coordinates)
ut.set_o_data(adata, f"{prefix}_y_{suffix}", y_coordinates)
def project_atlas_to_query(
*,
adata: AnnData,
qdata: AnnData,
weights: ut.ProperMatrix,
property_name: str,
formatter: Optional[Callable[[Any], Any]] = None,
to_property_name: Optional[str] = None,
method: Callable[[ut.Vector, ut.Vector], Any] = ut.highest_weight,
) -> None:
"""
Project the value of a property from per-observation atlas data to per-observation query data.
The input annotated ``adata`` is expected to contain a per-observation (cell) annotation named ``property_name``.
Given the ``weights`` matrix, where each row specifies the weights of the atlas metacells used to project a single
query metacell, this will generate a new per-observation (group) annotation in ``qdata``, named ``to_property_name``
(by default, the same as ``property_name``), containing the aggregated value of the property of all the observations
(cells) that belong to the group.
The aggregation method (by default, :py:func:`metacells.utilities.computation.highest_weight`) is any function
taking two array, weights and values, and returning a single value.
"""
if to_property_name is None:
to_property_name = property_name
property_of_atlas_metacells = ut.get_o_numpy(adata, property_name, formatter=formatter)
property_of_query_metacells = []
for query_metacell_index in range(qdata.n_obs):
metacell_weights = ut.to_numpy_vector(weights[query_metacell_index, :])
metacell_mask = metacell_weights > 0
metacell_weights = ut.to_numpy_vector(metacell_weights[metacell_mask])
metacell_values = property_of_atlas_metacells[metacell_mask]
property_of_query_metacells.append(method(metacell_weights, metacell_values))
ut.set_o_data(qdata, to_property_name, np.array(property_of_query_metacells))
def convey_obs_to_obs(
*,
adata: AnnData,
bdata: AnnData,
property_name: str,
formatter: Optional[Callable[[Any], Any]] = None,
to_property_name: Optional[str] = None,
default: Any = None,
) -> None:
"""
Project the value of a property from one annotated data to another.
The observation names are expected to be compatible between ``adata`` and ``bdata``. The
annotated ``adata`` is expected to contain a per-observation (cell) annotation named
``property_name``.
This will generate a new per-observation (cell) annotation in ``bdata``, named
``to_property_name`` (by default, the same as ``property_name``), containing the value of the
observation with the same name in ``adata``. If no such observation exists, the ``default``
value is used.
"""
if to_property_name is None:
to_property_name = property_name
property_of_from = ut.get_o_numpy(adata,
property_name,
formatter=formatter)
property_of_name = {
name: property_of_from[index]
for index, name in enumerate(adata.obs_names)
}
property_of_to = np.array(
[property_of_name.get(name, default) for name in bdata.obs_names])
ut.set_o_data(bdata, to_property_name, property_of_to)
def split_group(group_index: int) -> Tuple[ut.NumpyVector, ut.NumpyVector]:
group_cells_mask = group_of_cells == group_index
assert np.any(group_cells_mask)
name = f".{group}-{group_index}/{groups_count}"
gdata = ut.slice(adata,
name=name,
top_level=False,
obs=group_cells_mask,
track_obs="complete_cell_index")
target_metacell_size = (gdata.n_obs + 1) // 2
compute_direct_metacells(
gdata,
what,
feature_downsample_min_samples=feature_downsample_min_samples,
feature_downsample_min_cell_quantile=
feature_downsample_min_cell_quantile,
feature_downsample_max_cell_quantile=
feature_downsample_max_cell_quantile,
feature_min_gene_total=feature_min_gene_total,
feature_min_gene_top3=feature_min_gene_top3,
feature_min_gene_relative_variance=
feature_min_gene_relative_variance,
forbidden_gene_names=forbidden_gene_names,
forbidden_gene_patterns=forbidden_gene_patterns,
cells_similarity_value_normalization=
cells_similarity_value_normalization,
cells_similarity_log_data=cells_similarity_log_data,
cells_similarity_method=cells_similarity_method,
target_metacell_size=target_metacell_size,
max_cell_size=max_cell_size,
max_cell_size_factor=max_cell_size_factor,
cell_sizes=None,
knn_k=target_metacell_size,
min_knn_k=target_metacell_size,
knn_balanced_ranks_factor=knn_balanced_ranks_factor,
knn_incoming_degree_factor=knn_incoming_degree_factor,
knn_outgoing_degree_factor=knn_outgoing_degree_factor,
min_seed_size_quantile=min_seed_size_quantile,
max_seed_size_quantile=max_seed_size_quantile,
candidates_cooldown_pass=candidates_cooldown_pass,
candidates_cooldown_node=candidates_cooldown_node,
candidates_min_split_size_factor=None,
candidates_max_merge_size_factor=None,
candidates_min_metacell_cells=1,
must_complete_cover=True,
random_seed=random_seed,
)
direct_groups = ut.get_o_numpy(gdata, "metacell")
zero_count = np.sum(direct_groups == 0)
one_count = np.sum(direct_groups == 1)
ut.log_calc(f"group: {group_index} size: {len(direct_groups)} "
f"split into: {zero_count} + {one_count}")
assert zero_count + one_count == len(direct_groups)
assert zero_count > 0
assert one_count > 0
return (group_cells_mask, group_index + groups_count * direct_groups)
def convey_obs_obs_to_group_group(
*,
adata: AnnData,
gdata: AnnData,
group: str,
property_name: str,
formatter: Optional[Callable[[Any], Any]] = None,
to_property_name: Optional[str] = None,
method: Callable[[ut.Matrix], Any] = ut.nanmean_matrix,
) -> None:
"""
Project the value of a property from per-observation-per-observation data to per-group-per-group
data.
The input annotated ``adata`` is expected to contain a per-observation-per-observation (cell)
annotation named ``property_name`` and also a per-observation annotation named ``group`` which
identifies the group each observation (cell) belongs to, which must be an integer.
This will generate a new per-observation-per-observation (group) annotation in ``gdata``, named
``to_property_name`` (by default, the same as ``property_name``), containing the aggregated
value of the property of all the observations (cells) that belong to the group.
The aggregation method (by default, :py:func:`metacells.utilities.computation.nanmean_matrix`)
is any function taking a matrix of values and returning a single value.
"""
if to_property_name is None:
to_property_name = property_name
group_of_obs = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
property_of_obs_obs = ut.get_oo_proper(adata,
property_name,
formatter=formatter)
assert gdata.n_obs == (np.max(group_of_obs) + 1)
property_of_group_group = np.empty(
(gdata.n_obs, gdata.n_obs), dtype=ut.shaped_dtype(property_of_obs_obs))
# TODO: This is a slow implementation.
for row_group in range(gdata.n_obs):
row_cells = np.where(group_of_obs == row_group)[0]
assert len(row_cells) > 0
for column_group in range(gdata.n_obs):
column_cells = np.where(group_of_obs == column_group)[0]
assert len(column_cells) > 0
property_of_group_group[row_group, column_group] = method(
property_of_obs_obs[row_cells, :][:, column_cells])
ut.set_oo_data(gdata, to_property_name, property_of_group_group)
def convey_obs_to_group(
*,
adata: AnnData,
gdata: AnnData,
group: str,
property_name: str,
formatter: Optional[Callable[[Any], Any]] = None,
to_property_name: Optional[str] = None,
method: Callable[[ut.Vector], Any] = ut.most_frequent,
) -> None:
"""
Project the value of a property from per-observation data to per-group data.
The input annotated ``adata`` is expected to contain a per-observation (cell) annotation named
``property_name`` and also a per-observation annotation named ``group`` which identifies the
group each observation (cell) belongs to, which must be an integer.
This will generate a new per-observation (group) annotation in ``gdata``, named
``to_property_name`` (by default, the same as ``property_name``), containing the aggregated
value of the property of all the observations (cells) that belong to the group.
The aggregation method (by default, :py:func:`metacells.utilities.computation.most_frequent`) is
any function taking an array of values and returning a single value.
"""
if to_property_name is None:
to_property_name = property_name
group_of_obs = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
property_of_obs = ut.get_o_numpy(adata, property_name, formatter=formatter)
assert gdata.n_obs == (np.max(group_of_obs) + 1)
property_of_group = np.array([
method(property_of_obs[group_of_obs == group])
for group in range(gdata.n_obs)
])
ut.set_o_data(gdata, to_property_name, property_of_group)
def convey_group_to_obs(
*,
adata: AnnData,
gdata: AnnData,
group: str,
property_name: str,
formatter: Optional[Callable[[Any], Any]] = None,
to_property_name: Optional[str] = None,
default: Any = None,
) -> None:
"""
Project the value of a property from per-group data to per-observation data.
The input annotated ``gdata`` is expected to contain a per-observation (group) annotation named
``property_name``. The input annotated ``adata`` is expected to contain a per-observation
annotation named ``group`` which identifies the group each observation (cell) belongs to.
This will generate a new per-observation (cell) annotation in ``adata``, named
``to_property_name`` (by default, the same as ``property_name``), containing the value of the
property for the group it belongs to. If the ``group`` annotation contains a negative number
instead of a valid group index, the ``default`` value is used.
"""
if to_property_name is None:
to_property_name = property_name
group_of_obs = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
property_of_group = ut.get_o_numpy(gdata,
property_name,
formatter=formatter)
property_of_obs = np.array([
default if group < 0 else property_of_group[group]
for group in group_of_obs
])
ut.set_o_data(adata, to_property_name, property_of_obs)
def compute_outliers_matches(
what: Union[str, ut.Matrix] = "__x__",
*,
adata: AnnData,
gdata: AnnData,
group: Union[str, ut.Vector] = "metacell",
similar: str = "similar",
value_normalization: float = pr.outliers_value_normalization,
reproducible: bool,
) -> None:
"""
Given an assignment of observations (cells) to groups (metacells), compute for each outlier the "most similar"
group.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where ``what`` is a
per-variable-per-observation matrix or the name of a per-variable-per-observation annotation containing such a
matrix.
In addition, ``gdata`` is assumed to have one observation for each group, and use the same genes as ``adata``.
**Returns**
Sets the following in ``adata``:
Per-Observation (Cell) Annotations
``similar`` (default: {similar})
For each observation (cell), the index of the "most similar" group.
**Computation Parameters**
1. Compute the log2 of the fraction of each gene in each of the outlier cells and the group metacells using
the ``value_normalization`` (default: {value_normalization}).
2. Cross-correlate each of the outlier cells with each of the group metacells, in a ``reproducible`` manner.
"""
group_of_cells = ut.get_o_numpy(adata, group)
outliers_mask = group_of_cells < 0
odata = ut.slice(adata, obs=outliers_mask)
outliers_data = ut.get_vo_proper(odata, what, layout="row_major")
groups_data = ut.get_vo_proper(gdata, what, layout="row_major")
outliers_fractions = ut.fraction_by(outliers_data, by="row")
groups_fractions = ut.fraction_by(groups_data, by="row")
outliers_fractions = ut.to_numpy_matrix(outliers_fractions)
groups_fractions = ut.to_numpy_matrix(groups_fractions)
outliers_fractions += value_normalization
groups_fractions += value_normalization
outliers_log_fractions = np.log2(outliers_fractions,
out=outliers_fractions)
groups_log_fractions = np.log2(groups_fractions, out=groups_fractions)
outliers_groups_correlation = ut.cross_corrcoef_rows(
outliers_log_fractions,
groups_log_fractions,
reproducible=reproducible)
outliers_similar_group_indices = np.argmax(outliers_groups_correlation,
axis=1)
assert len(outliers_similar_group_indices) == odata.n_obs
cells_similar_group_indices = np.full(adata.n_obs, -1, dtype="int32")
cells_similar_group_indices[outliers_mask] = outliers_similar_group_indices
ut.set_o_data(adata, similar, cells_similar_group_indices)
def compute_direct_metacells( # pylint: disable=too-many-statements,too-many-branches
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
feature_downsample_min_samples: int = pr.feature_downsample_min_samples,
feature_downsample_min_cell_quantile: float = pr.feature_downsample_min_cell_quantile,
feature_downsample_max_cell_quantile: float = pr.feature_downsample_max_cell_quantile,
feature_min_gene_total: Optional[int] = pr.feature_min_gene_total,
feature_min_gene_top3: Optional[int] = pr.feature_min_gene_top3,
feature_min_gene_relative_variance: Optional[float] = pr.feature_min_gene_relative_variance,
feature_gene_names: Optional[Collection[str]] = None,
feature_gene_patterns: Optional[Collection[Union[str, Pattern]]] = None,
forbidden_gene_names: Optional[Collection[str]] = None,
forbidden_gene_patterns: Optional[Collection[Union[str, Pattern]]] = None,
cells_similarity_value_normalization: float = pr.cells_similarity_value_normalization,
cells_similarity_log_data: bool = pr.cells_similarity_log_data,
cells_similarity_method: str = pr.cells_similarity_method,
target_metacell_size: float = pr.target_metacell_size,
max_cell_size: Optional[float] = pr.max_cell_size,
max_cell_size_factor: Optional[float] = pr.max_cell_size_factor,
cell_sizes: Optional[Union[str, ut.Vector]] = pr.cell_sizes,
knn_k: Optional[int] = pr.knn_k,
min_knn_k: Optional[int] = pr.min_knn_k,
knn_balanced_ranks_factor: float = pr.knn_balanced_ranks_factor,
knn_incoming_degree_factor: float = pr.knn_incoming_degree_factor,
knn_outgoing_degree_factor: float = pr.knn_outgoing_degree_factor,
candidates_cell_seeds: Optional[Union[str, ut.Vector]] = None,
min_seed_size_quantile: float = pr.min_seed_size_quantile,
max_seed_size_quantile: float = pr.max_seed_size_quantile,
candidates_cooldown_pass: float = pr.cooldown_pass,
candidates_cooldown_node: float = pr.cooldown_node,
candidates_cooldown_phase: float = pr.cooldown_phase,
candidates_min_split_size_factor: Optional[float] = pr.candidates_min_split_size_factor,
candidates_max_merge_size_factor: Optional[float] = pr.candidates_max_merge_size_factor,
candidates_min_metacell_cells: Optional[int] = pr.min_metacell_cells,
candidates_max_split_min_cut_strength: Optional[float] = pr.max_split_min_cut_strength,
candidates_min_cut_seed_cells: Optional[int] = pr.min_cut_seed_cells,
must_complete_cover: bool = False,
deviants_min_gene_fold_factor: float = pr.deviants_min_gene_fold_factor,
deviants_abs_folds: bool = pr.deviants_abs_folds,
deviants_max_gene_fraction: Optional[float] = pr.deviants_max_gene_fraction,
deviants_max_cell_fraction: Optional[float] = pr.deviants_max_cell_fraction,
dissolve_min_robust_size_factor: Optional[float] = pr.dissolve_min_robust_size_factor,
dissolve_min_convincing_size_factor: Optional[float] = pr.dissolve_min_convincing_size_factor,
dissolve_min_convincing_gene_fold_factor: float = pr.dissolve_min_convincing_gene_fold_factor,
dissolve_min_metacell_cells: int = pr.dissolve_min_metacell_cells,
random_seed: int = pr.random_seed,
) -> AnnData:
"""
Directly compute metacells using ``what`` (default: {what}) data.
This directly computes the metacells on the whole data. Like any method that directly looks at
the whole data at once, the amount of CPU and memory needed becomes unreasonable when the data
size grows. Above O(10,000) you are much better off using the divide-and-conquer method.
.. note::
The current implementation is naive in that it computes the full dense N^2 correlation
matrix, and only then extracts the sparse graph out of it. We actually need two copies where
each requires 4 bytes per entry, so for O(100,000) cells, we have storage of
O(100,000,000,000). In addition, the implementation is serial for the graph clustering
phases.
It is possible to mitigate this by fusing the correlations phase and the graph generation
phase, parallelizing the result, and also (somehow) parallelizing the graph clustering
phase. This might increase the "reasonable" size for the direct approach to O(100,000).
We have decided not to invest in this direction since it won't allow us to push the size to
O(1,000,000) and above. Instead we provide the divide-and-conquer method, which easily
scales to O(1,000,000) on a single multi-core server, and to "unlimited" size if we further
enhance the implementation to use a distributed compute cluster of such servers.
.. todo::
Should :py:func:`compute_direct_metacells` avoid computing the graph and partition it for a
very small number of cells?
**Input**
The presumably "clean" annotated ``adata``, where the observations are cells and the variables
are genes, where ``what`` is a per-variable-per-observation matrix or the name of a
per-variable-per-observation annotation containing such a matrix.
**Returns**
Sets the following annotations in ``adata``:
Variable (Gene) Annotations
``high_total_gene``
A boolean mask of genes with "high" expression level.
``high_relative_variance_gene``
A boolean mask of genes with "high" normalized variance, relative to other genes with a
similar expression level.
``forbidden_gene``
A boolean mask of genes which are forbidden from being chosen as "feature" genes based
on their name.
``feature_gene``
A boolean mask of the "feature" genes.
``gene_deviant_votes``
The number of cells each gene marked as deviant (if zero, the gene did not mark any cell
as deviant). This will be zero for non-"feature" genes.
Observation (Cell) Annotations
``seed``
The index of the seed metacell each cell was assigned to to. This is ``-1`` for
non-"clean" cells.
``candidate``
The index of the candidate metacell each cell was assigned to to. This is ``-1`` for
non-"clean" cells.
``cell_deviant_votes``
The number of genes that were the reason the cell was marked as deviant (if zero, the
cell is not deviant).
``dissolved``
A boolean mask of the cells contained in a dissolved metacell.
``metacell``
The integer index of the metacell each cell belongs to. The metacells are in no
particular order. Cells with no metacell assignment ("outliers") are given a metacell
index of ``-1``.
``outlier``
A boolean mask of the cells contained in no metacell.
**Computation Parameters**
1. Invoke :py:func:`metacells.pipeline.feature.extract_feature_data` to extract "feature" data
from the clean data, using the
``feature_downsample_min_samples`` (default: {feature_downsample_min_samples}),
``feature_downsample_min_cell_quantile`` (default: {feature_downsample_min_cell_quantile}),
``feature_downsample_max_cell_quantile`` (default: {feature_downsample_max_cell_quantile}),
``feature_min_gene_total`` (default: {feature_min_gene_total}), ``feature_min_gene_top3``
(default: {feature_min_gene_top3}), ``feature_min_gene_relative_variance`` (default:
{feature_min_gene_relative_variance}), ``feature_gene_names`` (default:
{feature_gene_names}), ``feature_gene_patterns`` (default: {feature_gene_patterns}),
``forbidden_gene_names`` (default: {forbidden_gene_names}), ``forbidden_gene_patterns``
(default: {forbidden_gene_patterns}) and ``random_seed`` (default: {random_seed}) to make
this replicable.
2. Compute the fractions of each variable in each cell, and add the
``cells_similarity_value_normalization`` (default: {cells_similarity_value_normalization}) to
it.
3. If ``cells_similarity_log_data`` (default: {cells_similarity_log_data}), invoke the
:py:func:`metacells.utilities.computation.log_data` function to compute the log (base 2) of
the data.
4. Invoke :py:func:`metacells.tools.similarity.compute_obs_obs_similarity` to compute the
similarity between each pair of cells, using the
``cells_similarity_method`` (default: {cells_similarity_method}).
5. Invoke :py:func:`metacells.pipeline.collect.compute_effective_cell_sizes` using
``max_cell_size`` (default: {max_cell_size}), ``max_cell_size_factor`` (default:
{max_cell_size_factor}) and ``cell_sizes`` (default: {cell_sizes}) to get the effective cell
sizes to use.
5. Invoke :py:func:`metacells.tools.knn_graph.compute_obs_obs_knn_graph` to compute a
K-Nearest-Neighbors graph, using the
``knn_balanced_ranks_factor`` (default: {knn_balanced_ranks_factor}),
``knn_incoming_degree_factor`` (default: {knn_incoming_degree_factor})
and
``knn_outgoing_degree_factor`` (default: {knn_outgoing_degree_factor}).
If ``knn_k`` (default: {knn_k}) is not specified, then it is
chosen to be the median number of cells required to reach the target metacell size,
but at least ``min_knn_k`` (default: {min_knn_k}).
6. Invoke :py:func:`metacells.tools.candidates.compute_candidate_metacells` to compute
the candidate metacells, using the
``candidates_cell_seeds`` (default: {candidates_cell_seeds}),
``min_seed_size_quantile`` (default: {min_seed_size_quantile}),
``max_seed_size_quantile`` (default: {max_seed_size_quantile}),
``candidates_cooldown_pass`` (default: {candidates_cooldown_pass}),
``candidates_cooldown_node`` (default: {candidates_cooldown_node}),
``candidates_cooldown_phase`` (default: {candidates_cooldown_phase}),
``candidates_min_split_size_factor`` (default: {candidates_min_split_size_factor}),
``candidates_max_merge_size_factor`` (default: {candidates_max_merge_size_factor}),
``candidates_min_metacell_cells`` (default: {candidates_min_metacell_cells}),
and
``random_seed`` (default: {random_seed})
to make this replicable. This tries to build metacells of the
``target_metacell_size`` (default: {target_metacell_size})
using the effective cell sizes.
7. Unless ``must_complete_cover`` (default: {must_complete_cover}), invoke
:py:func:`metacells.tools.deviants.find_deviant_cells` to remove deviants from the candidate
metacells, using the
``deviants_min_gene_fold_factor`` (default: {deviants_min_gene_fold_factor}),
``deviants_abs_folds`` (default: {deviants_abs_folds}),
``deviants_max_gene_fraction`` (default: {deviants_max_gene_fraction})
and
``deviants_max_cell_fraction`` (default: {deviants_max_cell_fraction}).
8. Unless ``must_complete_cover`` (default: {must_complete_cover}), invoke
:py:func:`metacells.tools.dissolve.dissolve_metacells` to dissolve small unconvincing
metacells, using the same
``target_metacell_size`` (default: {target_metacell_size}),
and the effective cell sizes
and the
``dissolve_min_robust_size_factor`` (default: {dissolve_min_robust_size_factor}),
``dissolve_min_convincing_size_factor`` (default: {dissolve_min_convincing_size_factor}),
``dissolve_min_convincing_gene_fold_factor`` (default: {dissolve_min_convincing_size_factor})
and
``dissolve_min_metacell_cells`` (default: ``dissolve_min_metacell_cells``).
"""
fdata = extract_feature_data(
adata,
what,
top_level=False,
downsample_min_samples=feature_downsample_min_samples,
downsample_min_cell_quantile=feature_downsample_min_cell_quantile,
downsample_max_cell_quantile=feature_downsample_max_cell_quantile,
min_gene_relative_variance=feature_min_gene_relative_variance,
min_gene_total=feature_min_gene_total,
min_gene_top3=feature_min_gene_top3,
forced_gene_names=feature_gene_names,
forced_gene_patterns=feature_gene_patterns,
forbidden_gene_names=forbidden_gene_names,
forbidden_gene_patterns=forbidden_gene_patterns,
random_seed=random_seed,
)
if fdata is None:
raise ValueError("Empty feature data, giving up")
effective_cell_sizes, max_cell_size, _cell_scale_factors = compute_effective_cell_sizes(
adata, max_cell_size=max_cell_size, max_cell_size_factor=max_cell_size_factor, cell_sizes=cell_sizes
)
ut.log_calc("effective_cell_sizes", effective_cell_sizes, formatter=ut.sizes_description)
if max_cell_size is not None:
if candidates_min_metacell_cells is not None:
target_metacell_size = max(target_metacell_size, max_cell_size * candidates_min_metacell_cells)
if dissolve_min_metacell_cells is not None:
target_metacell_size = max(target_metacell_size, max_cell_size * dissolve_min_metacell_cells)
if candidates_min_metacell_cells is not None or dissolve_min_metacell_cells is not None:
ut.log_calc("target_metacell_size", target_metacell_size)
data = ut.get_vo_proper(fdata, "downsampled", layout="row_major")
data = ut.to_numpy_matrix(data, copy=True)
if cells_similarity_value_normalization > 0:
data += cells_similarity_value_normalization
if cells_similarity_log_data:
data = ut.log_data(data, base=2)
if knn_k is None:
if effective_cell_sizes is None:
median_cell_size = 1.0
else:
median_cell_size = float(np.median(effective_cell_sizes))
knn_k = int(round(target_metacell_size / median_cell_size))
if min_knn_k is not None:
knn_k = max(knn_k, min_knn_k)
if knn_k == 0:
ut.log_calc("knn_k: 0 (too small, try single metacell)")
ut.set_o_data(fdata, "candidate", np.full(fdata.n_obs, 0, dtype="int32"), formatter=lambda _: "* <- 0")
elif knn_k >= fdata.n_obs:
ut.log_calc(f"knn_k: {knn_k} (too large, try single metacell)")
ut.set_o_data(fdata, "candidate", np.full(fdata.n_obs, 0, dtype="int32"), formatter=lambda _: "* <- 0")
else:
ut.log_calc("knn_k", knn_k)
tl.compute_obs_obs_similarity(fdata, data, method=cells_similarity_method, reproducible=(random_seed != 0))
tl.compute_obs_obs_knn_graph(
fdata,
k=knn_k,
balanced_ranks_factor=knn_balanced_ranks_factor,
incoming_degree_factor=knn_incoming_degree_factor,
outgoing_degree_factor=knn_outgoing_degree_factor,
)
tl.compute_candidate_metacells(
fdata,
target_metacell_size=target_metacell_size,
cell_sizes=effective_cell_sizes,
cell_seeds=candidates_cell_seeds,
min_seed_size_quantile=min_seed_size_quantile,
max_seed_size_quantile=max_seed_size_quantile,
cooldown_pass=candidates_cooldown_pass,
cooldown_node=candidates_cooldown_node,
cooldown_phase=candidates_cooldown_phase,
min_split_size_factor=candidates_min_split_size_factor,
max_merge_size_factor=candidates_max_merge_size_factor,
min_metacell_cells=candidates_min_metacell_cells,
max_split_min_cut_strength=candidates_max_split_min_cut_strength,
min_cut_seed_cells=candidates_min_cut_seed_cells,
must_complete_cover=must_complete_cover,
random_seed=random_seed,
)
ut.set_oo_data(adata, "obs_similarity", ut.get_oo_proper(fdata, "obs_similarity"))
ut.set_oo_data(adata, "obs_outgoing_weights", ut.get_oo_proper(fdata, "obs_outgoing_weights"))
seed_of_cells = ut.get_o_numpy(fdata, "seed", formatter=ut.groups_description)
ut.set_o_data(adata, "seed", seed_of_cells, formatter=ut.groups_description)
candidate_of_cells = ut.get_o_numpy(fdata, "candidate", formatter=ut.groups_description)
ut.set_o_data(adata, "candidate", candidate_of_cells, formatter=ut.groups_description)
if must_complete_cover:
assert np.min(candidate_of_cells) == 0
deviant_votes_of_genes = np.zeros(adata.n_vars, dtype="float32")
deviant_votes_of_cells = np.zeros(adata.n_obs, dtype="float32")
dissolved_of_cells = np.zeros(adata.n_obs, dtype="bool")
ut.set_v_data(adata, "gene_deviant_votes", deviant_votes_of_genes, formatter=ut.mask_description)
ut.set_o_data(adata, "cell_deviant_votes", deviant_votes_of_cells, formatter=ut.mask_description)
ut.set_o_data(adata, "dissolved", dissolved_of_cells, formatter=ut.mask_description)
ut.set_o_data(adata, "metacell", candidate_of_cells, formatter=ut.groups_description)
else:
tl.find_deviant_cells(
adata,
candidates=candidate_of_cells,
min_gene_fold_factor=deviants_min_gene_fold_factor,
abs_folds=deviants_abs_folds,
max_gene_fraction=deviants_max_gene_fraction,
max_cell_fraction=deviants_max_cell_fraction,
)
tl.dissolve_metacells(
adata,
candidates=candidate_of_cells,
target_metacell_size=target_metacell_size,
cell_sizes=effective_cell_sizes,
min_robust_size_factor=dissolve_min_robust_size_factor,
min_convincing_size_factor=dissolve_min_convincing_size_factor,
min_convincing_gene_fold_factor=dissolve_min_convincing_gene_fold_factor,
min_metacell_cells=dissolve_min_metacell_cells,
)
metacell_of_cells = ut.get_o_numpy(adata, "metacell", formatter=ut.groups_description)
outlier_of_cells = metacell_of_cells < 0
ut.set_o_data(adata, "outlier", outlier_of_cells, formatter=ut.mask_description)
return fdata
def filter_data( # pylint: disable=dangerous-default-value
adata: AnnData,
obs_masks: List[str] = [],
var_masks: List[str] = [],
*,
mask_obs: Optional[str] = None,
mask_var: Optional[str] = None,
invert_obs: bool = False,
invert_var: bool = False,
track_obs: Optional[str] = None,
track_var: Optional[str] = None,
name: Optional[str] = None,
top_level: bool = True,
) -> Optional[Tuple[AnnData, ut.PandasSeries, ut.PandasSeries]]:
"""
Filter (slice) the data based on previously-computed masks.
For example, it is useful to discard cell-cycle genes, cells which have too few UMIs for
meaningful analysis, etc. In general, the "best" filter depends on the data set.
This function makes it easy to combine different pre-computed per-observation (cell) and
per-variable (gene) boolean mask annotations into a final overall inclusion mask, and slice the
data accordingly, while tracking the base index of the cells and genes in the filtered data.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes.
**Returns**
An annotated data containing a subset of the observations (cells) and variables (genes).
If no observations and/or no variables were selected by the filter, returns ``None``.
If ``name`` is not specified, the returned data will be unnamed. Otherwise, if the name starts
with a ``.``, it will be appended to the current name (if any). Otherwise, ``name`` is the new
name.
If ``mask_obs`` and/or ``mask_var`` are specified, store the mask of the selected data as a
per-observation and/or per-variable annotation of the full ``adata``.
If ``track_obs`` and/or ``track_var`` are specified, store the original indices of the selected
data as a per-observation and/or per-variable annotation of the result data.
**Computation Parameters**
1. Combine the masks in ``obs_masks`` and/or ``var_masks`` using
:py:func:`metacells.tools.mask.combine_masks` passing it ``invert_obs`` and ``invert_var``,
and ``mask_obs`` and ``mask_var`` as the ``to`` parameter. If either list of masks is empty,
use the full mask.
2. If the obtained masks for either the observations or variables is empty, return ``None``.
Otherwise, return a slice of the full data containing just the observations and variables
specified by the final masks.
"""
if len(obs_masks) == 0:
obs_mask = np.full(adata.n_obs, True, dtype="bool")
if mask_obs is not None:
ut.set_o_data(adata, mask_obs, obs_mask)
else:
mask = combine_masks(adata, obs_masks, invert=invert_obs, to=mask_obs)
if mask is None:
assert mask_obs is not None
obs_mask = ut.get_o_numpy(
adata, mask_obs, formatter=ut.mask_description) > 0
else:
obs_mask = ut.to_numpy_vector(mask, only_extract=True) > 0
if len(var_masks) == 0:
var_mask = np.full(adata.n_vars, True, dtype="bool")
if mask_var is not None:
ut.set_o_data(adata, mask_var, var_mask)
else:
mask = combine_masks(adata, var_masks, invert=invert_var, to=mask_var)
if mask is None:
assert mask_var is not None
var_mask = ut.get_v_numpy(
adata, mask_var, formatter=ut.mask_description) > 0
else:
var_mask = ut.to_numpy_vector(mask, only_extract=True) > 0
if not np.any(obs_mask) or not np.any(var_mask):
return None
fdata = ut.slice(adata,
name=name,
top_level=top_level,
obs=obs_mask,
vars=var_mask,
track_obs=track_obs,
track_var=track_var)
return (
fdata,
ut.to_pandas_series(obs_mask, index=adata.obs_names),
ut.to_pandas_series(var_mask, index=adata.var_names),
)
def compute_distinct_folds(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
normalization: float = 0,
inplace: bool = True,
) -> Optional[ut.PandasFrame]:
"""
Compute for each observation (cell) and each variable (gene) how much is the ``what`` (default:
{what}) value different from the overall population.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Per-Observation-Per-Variable (Cell-Gene) Annotations:
``distinct_ratio``
For each gene in each cell, the log based 2 of the ratio between the fraction of the
gene in the cell and the fraction of the gene in the overall population (sum of cells).
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas frame (indexed by the observation and
distinct gene rank).
**Computation Parameters**
1. Compute, for each gene, the fraction of the gene's values out of the total sum of the values
(that is, the mean fraction of the gene's expression in the population).
2. Compute, for each cell, for each gene, the fraction of the gene's value out of the sum
of the values in the cell (that is, the fraction of the gene's expression in the cell).
3. Divide the two to the distinct ratio (that is, how much the gene's expression in the cell is
different from the overall population), first adding the ``normalization`` (default:
{normalization}) to both.
4. Compute the log (base 2) of the result and use it as the fold factor.
"""
columns_data = ut.get_vo_proper(adata, what, layout="column_major")
fractions_of_genes_in_data = ut.fraction_per(columns_data, per="column")
fractions_of_genes_in_data += normalization
total_umis_of_cells = ut.get_o_numpy(adata, what, sum=True)
total_umis_of_cells[total_umis_of_cells == 0] = 1
rows_data = ut.get_vo_proper(adata, what, layout="row_major")
fraction_of_genes_in_cells = ut.to_numpy_matrix(
rows_data) / total_umis_of_cells[:, np.newaxis]
fraction_of_genes_in_cells += normalization
zeros_mask = fractions_of_genes_in_data <= 0
fractions_of_genes_in_data[zeros_mask] = -1
fraction_of_genes_in_cells[:, zeros_mask] = -1
ratio_of_genes_in_cells = fraction_of_genes_in_cells
ratio_of_genes_in_cells /= fractions_of_genes_in_data
assert np.min(np.min(ratio_of_genes_in_cells)) > 0
fold_of_genes_in_cells = np.log2(ratio_of_genes_in_cells,
out=ratio_of_genes_in_cells)
if inplace:
ut.set_vo_data(adata, "distinct_fold", fold_of_genes_in_cells)
return None
return ut.to_pandas_frame(fold_of_genes_in_cells,
index=adata.obs_names,
columns=adata.var_names)
def find_deviant_cells(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
candidates: Union[str, ut.Vector] = "candidate",
min_gene_fold_factor: float = pr.deviants_min_gene_fold_factor,
abs_folds: bool = pr.deviants_abs_folds,
max_gene_fraction: Optional[float] = pr.deviants_max_gene_fraction,
max_cell_fraction: Optional[float] = pr.deviants_max_cell_fraction,
inplace: bool = True,
) -> Optional[Tuple[ut.PandasSeries, ut.PandasSeries]]:
"""
Find cells which are have significantly different gene expression from the metacells they are
belong to based on ``what`` (default: {what}) data.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Observation (Cell) Annotations
``cell_deviant_votes``
The number of genes that were the reason the cell was marked as deviant (if zero, the
cell is not deviant).
Variable (Gene) Annotations
``gene_deviant_votes``
The number of cells each gene marked as deviant (if zero, the gene did not mark any cell
as deviant).
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as two pandas series (indexed by the observation and
variable names).
**Computation Parameters**
Intuitively, we first select some fraction of the genes which were least predictable compared to
the mean expression in the candidate metacells. We then mark as deviants some fraction of the
cells whose expression of these genes was least predictable compared to the mean expression in
the candidate metacells. Operationally:
1. Compute for each candidate metacell the mean fraction of the UMIs expressed by each gene.
Scale this by each cell's total UMIs to compute the expected number of UMIs for each cell.
Compute the fold factor log2((actual UMIs + 1) / (expected UMIs + 1)) for each gene for each
cell.
2. Ignore all fold factors less than the ``min_gene_fold_factor`` (default: {min_gene_fold_factor}). If
``abs_folds`` (default: {abs_folds}), consider the absolute fold factors. Count the number of genes which have a
fold factor above this minimum in at least one cell. If the fraction of such genes is above ``max_gene_fraction``
(default: {max_gene_fraction}), then raise the minimal gene fold factor such that at most this fraction of genes
remain.
3. For each remaining gene, rank all the cells where it is expressed above the min fold
factor. Give an artificial maximum rank to all cells with fold factor 0, that is, below the
minimum.
4. For each cell, compute the minimal rank it has in any of these genes. That is, if a cell has
a rank of 1, it means that it has at least one gene whose expression fold factor is the worst
(highest) across all cells (and is also above the minimum).
5. Select as deviants all cells whose minimal rank is below the artificial maximum rank, that
is, which contain at least one gene whose expression fold factor is high relative to the rest
of the cells. If the fraction of such cells is higher than ``max_cell_fraction`` (default:
{max_cell_fraction}), reduce the maximal rank such that at most this fraction of cells are
selected as deviants.
"""
if max_gene_fraction is None:
max_gene_fraction = 1
if max_cell_fraction is None:
max_cell_fraction = 1
assert min_gene_fold_factor > 0
assert 0 < max_gene_fraction < 1
assert 0 < max_cell_fraction < 1
cells_count, genes_count = adata.shape
assert cells_count > 0
candidate_of_cells = ut.get_o_numpy(adata, candidates, formatter=ut.groups_description)
totals_of_cells = ut.get_o_numpy(adata, what, sum=True)
assert totals_of_cells.size == cells_count
data = ut.get_vo_proper(adata, what, layout="row_major")
list_of_fold_factors, list_of_cell_index_of_rows = _collect_fold_factors(
data=data,
candidate_of_cells=candidate_of_cells,
totals_of_cells=totals_of_cells,
min_gene_fold_factor=min_gene_fold_factor,
abs_folds=abs_folds,
)
fold_factors = _construct_fold_factors(cells_count, list_of_fold_factors, list_of_cell_index_of_rows)
if fold_factors is None:
votes_of_deviant_cells = np.zeros(adata.n_obs, dtype="int32")
votes_of_deviant_genes = np.zeros(adata.n_vars, dtype="int32")
else:
deviant_gene_indices = _filter_genes(
cells_count=cells_count,
genes_count=genes_count,
fold_factors=fold_factors,
min_gene_fold_factor=min_gene_fold_factor,
max_gene_fraction=max_gene_fraction,
)
deviant_genes_fold_ranks = _fold_ranks(
cells_count=cells_count, fold_factors=fold_factors, deviant_gene_indices=deviant_gene_indices
)
votes_of_deviant_cells, votes_of_deviant_genes = _filter_cells(
cells_count=cells_count,
genes_count=genes_count,
deviant_genes_fold_ranks=deviant_genes_fold_ranks,
deviant_gene_indices=deviant_gene_indices,
max_cell_fraction=max_cell_fraction,
)
if inplace:
ut.set_v_data(adata, "gene_deviant_votes", votes_of_deviant_genes, formatter=ut.mask_description)
ut.set_o_data(adata, "cell_deviant_votes", votes_of_deviant_cells, formatter=ut.mask_description)
return None
ut.log_return("gene_deviant_votes", votes_of_deviant_genes, formatter=ut.mask_description)
ut.log_return("cell_deviant_votes", votes_of_deviant_cells, formatter=ut.mask_description)
return (
ut.to_pandas_series(votes_of_deviant_cells, index=adata.obs_names),
ut.to_pandas_series(votes_of_deviant_genes, index=adata.var_names),
)
def _compress_modules(
*,
adata_of_all_genes_of_all_cells: AnnData,
what: Union[str, ut.Matrix] = "__x__",
min_cells_of_modules: int,
max_cells_of_modules: int,
target_metacell_size: float,
min_modules_size_factor: float,
related_gene_indices_of_modules: List[List[int]],
rare_module_of_cells: ut.NumpyVector,
) -> List[List[int]]:
list_of_rare_gene_indices_of_modules: List[List[int]] = []
list_of_names_of_genes_of_modules: List[List[str]] = []
min_umis_of_modules = target_metacell_size * min_modules_size_factor
ut.log_calc("min_umis_of_modules", min_umis_of_modules)
total_all_genes_of_all_cells = ut.get_o_numpy(
adata_of_all_genes_of_all_cells, what, sum=True)
cell_counts_of_modules: List[int] = []
ut.log_calc("compress modules:")
modules_count = len(related_gene_indices_of_modules)
for module_index, gene_indices_of_module in enumerate(
related_gene_indices_of_modules):
if len(gene_indices_of_module) == 0:
continue
with ut.log_step(
"- module",
module_index,
formatter=lambda module_index: ut.progress_description(
modules_count, module_index, "module"),
):
module_cells_mask = rare_module_of_cells == module_index
module_cells_count = np.sum(module_cells_mask)
module_umis_count = np.sum(
total_all_genes_of_all_cells[module_cells_mask])
if module_cells_count < min_cells_of_modules:
if ut.logging_calc():
ut.log_calc("cells",
str(module_cells_count) + " (too few)")
rare_module_of_cells[module_cells_mask] = -1
continue
if module_cells_count > max_cells_of_modules:
if ut.logging_calc():
ut.log_calc("cells",
str(module_cells_count) + " (too many)")
rare_module_of_cells[module_cells_mask] = -1
continue
ut.log_calc("cells", module_cells_count)
if module_umis_count < min_umis_of_modules:
if ut.logging_calc():
ut.log_calc("UMIs", str(module_umis_count) + " (too few)")
rare_module_of_cells[module_cells_mask] = -1
continue
ut.log_calc("UMIs", module_umis_count)
next_module_index = len(list_of_rare_gene_indices_of_modules)
if module_index != next_module_index:
ut.log_calc("is reindexed to", next_module_index)
rare_module_of_cells[module_cells_mask] = next_module_index
module_index = next_module_index
next_module_index += 1
list_of_rare_gene_indices_of_modules.append(gene_indices_of_module)
if ut.logging_calc():
cell_counts_of_modules.append(np.sum(module_cells_mask))
list_of_names_of_genes_of_modules.append( #
sorted(adata_of_all_genes_of_all_cells.
var_names[gene_indices_of_module]))
if ut.logging_calc():
ut.log_calc("final modules:")
for module_index, (module_cells_count, module_gene_names) in enumerate(
zip(cell_counts_of_modules,
list_of_names_of_genes_of_modules)):
ut.log_calc(
f"- module: {module_index} cells: {module_cells_count} genes: {module_gene_names}"
) #
return list_of_rare_gene_indices_of_modules
def _related_genes( # pylint: disable=too-many-statements,too-many-branches
*,
adata_of_all_genes_of_all_cells: AnnData,
what: Union[str, ut.Matrix] = "__x__",
rare_gene_indices_of_modules: List[List[int]],
allowed_genes_mask: ut.NumpyVector,
min_genes_of_modules: int,
min_gene_maximum: int,
min_cells_of_modules: int,
max_cells_of_modules: int,
min_cell_module_total: int,
min_related_gene_fold_factor: float,
max_related_gene_increase_factor: float,
) -> List[List[int]]:
total_all_cells_umis_of_all_genes = ut.get_v_numpy(
adata_of_all_genes_of_all_cells, what, sum=True)
ut.log_calc("genes for modules:")
modules_count = 0
related_gene_indices_of_modules: List[List[int]] = []
rare_gene_indices_of_any: Set[int] = set()
for rare_gene_indices_of_module in rare_gene_indices_of_modules:
if len(rare_gene_indices_of_module) >= min_genes_of_modules:
rare_gene_indices_of_any.update(list(rare_gene_indices_of_module))
for rare_gene_indices_of_module in rare_gene_indices_of_modules:
if len(rare_gene_indices_of_module) < min_genes_of_modules:
continue
module_index = modules_count
modules_count += 1
with ut.log_step("- module", module_index):
ut.log_calc(
"rare_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[rare_gene_indices_of_module]))
adata_of_module_genes_of_all_cells = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.rare_gene",
vars=rare_gene_indices_of_module,
top_level=False,
)
total_module_genes_umis_of_all_cells = ut.get_o_numpy(
adata_of_module_genes_of_all_cells, what, sum=True)
mask_of_expressed_cells = total_module_genes_umis_of_all_cells > 0
expressed_cells_count = np.sum(mask_of_expressed_cells)
if expressed_cells_count > max_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"expressed_cells",
ut.mask_description(mask_of_expressed_cells) +
" (too many)")
continue
if expressed_cells_count < min_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"expressed_cells",
ut.mask_description(mask_of_expressed_cells) +
" (too few)")
continue
ut.log_calc("expressed_cells", mask_of_expressed_cells)
adata_of_all_genes_of_expressed_cells_of_module = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.rare_cell",
obs=mask_of_expressed_cells,
top_level=False,
)
total_expressed_cells_umis_of_all_genes = ut.get_v_numpy(
adata_of_all_genes_of_expressed_cells_of_module,
what,
sum=True)
data = ut.get_vo_proper(
adata_of_all_genes_of_expressed_cells_of_module,
what,
layout="column_major")
max_expressed_cells_umis_of_all_genes = ut.max_per(data,
per="column")
total_background_cells_umis_of_all_genes = (
total_all_cells_umis_of_all_genes -
total_expressed_cells_umis_of_all_genes)
expressed_cells_fraction_of_all_genes = total_expressed_cells_umis_of_all_genes / sum(
total_expressed_cells_umis_of_all_genes)
background_cells_fraction_of_all_genes = total_background_cells_umis_of_all_genes / sum(
total_background_cells_umis_of_all_genes)
mask_of_related_genes = (
allowed_genes_mask
& (max_expressed_cells_umis_of_all_genes >= min_gene_maximum)
& (expressed_cells_fraction_of_all_genes >=
background_cells_fraction_of_all_genes *
(2**min_related_gene_fold_factor)))
related_gene_indices = np.where(mask_of_related_genes)[0]
assert np.all(mask_of_related_genes[rare_gene_indices_of_module])
base_genes_of_all_cells_adata = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.base",
vars=rare_gene_indices_of_module)
total_base_genes_of_all_cells = ut.get_o_numpy(
base_genes_of_all_cells_adata, what, sum=True)
mask_of_strong_base_cells = total_base_genes_of_all_cells >= min_cell_module_total
count_of_strong_base_cells = np.sum(mask_of_strong_base_cells)
if ut.logging_calc():
ut.log_calc(
"candidate_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[related_gene_indices]))
ut.log_calc("base_strong_genes", count_of_strong_base_cells)
related_gene_indices_of_module = list(rare_gene_indices_of_module)
for gene_index in related_gene_indices:
if gene_index in rare_gene_indices_of_module:
continue
if gene_index in rare_gene_indices_of_any:
ut.log_calc(
f"- candidate gene {adata_of_all_genes_of_all_cells.var_names[gene_index]} "
f"belongs to another module")
continue
if gene_index not in rare_gene_indices_of_module:
related_gene_of_all_cells_adata = ut.slice(
adata_of_all_genes_of_all_cells,
name=
f".{adata_of_all_genes_of_all_cells.var_names[gene_index]}",
vars=np.array([gene_index]),
)
assert related_gene_of_all_cells_adata.n_vars == 1
total_related_genes_of_all_cells = ut.get_o_numpy(
related_gene_of_all_cells_adata, what, sum=True)
total_related_genes_of_all_cells += total_base_genes_of_all_cells
mask_of_strong_related_cells = total_related_genes_of_all_cells >= min_cell_module_total
count_of_strong_related_cells = np.sum(
mask_of_strong_related_cells)
ut.log_calc(
f"- candidate gene {adata_of_all_genes_of_all_cells.var_names[gene_index]} "
f"strong cells: {count_of_strong_related_cells} "
f"factor: {count_of_strong_related_cells / count_of_strong_base_cells}"
)
if count_of_strong_related_cells > max_related_gene_increase_factor * count_of_strong_base_cells:
continue
related_gene_indices_of_module.append(gene_index)
related_gene_indices_of_modules.append(
related_gene_indices_of_module) #
if ut.logging_calc():
ut.log_calc("related genes for modules:")
for module_index, related_gene_indices_of_module in enumerate(
related_gene_indices_of_modules):
ut.log_calc(
f"- module {module_index} related_gene_names",
sorted(adata_of_all_genes_of_all_cells.
var_names[related_gene_indices_of_module]),
)
return related_gene_indices_of_modules
def combine_masks( # pylint: disable=too-many-branches,too-many-statements
adata: AnnData,
masks: List[str],
*,
invert: bool = False,
to: Optional[str] = None,
) -> Optional[ut.PandasSeries]:
"""
Combine different pre-computed masks into a final overall mask.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes.
**Returns**
If ``to`` (default: {to}) is ``None``, returns the computed mask. Otherwise, sets the
mask as an annotation (per-variable or per-observation depending on the type of the combined masks).
**Computation Parameters**
1. For each of the mask in ``masks``, fetch it. Silently ignore missing masks if the name has a
``?`` suffix. Invert the mask if the name has a ``~`` prefix. If the name has a ``|`` prefix
(before the ``~`` prefix, if any), then bitwise-OR the mask into the OR mask, otherwise (or if
it has a ``&`` prefix), bitwise-AND the mask into the AND mask.
2. Combine (bitwise-AND) the AND mask and the OR mask into a single mask.
3. If ``invert`` (default: {invert}), invert the result combined mask.
"""
assert len(masks) > 0
per: Optional[str] = None
and_mask: Optional[ut.NumpyVector] = None
or_mask: Optional[ut.NumpyVector] = None
for mask_name in masks:
log_mask_name = mask_name
if mask_name[0] == "|":
is_or = True
mask_name = mask_name[1:]
else:
is_or = False
if mask_name[0] == "&":
mask_name = mask_name[1:]
if mask_name[0] == "~":
invert_mask = True
mask_name = mask_name[1:]
else:
invert_mask = False
if mask_name[-1] == "?":
must_exist = False
mask_name = mask_name[:-1]
else:
must_exist = True
if mask_name in adata.obs:
mask_per = "o"
mask = ut.get_o_numpy(
adata, mask_name, formatter=ut.mask_description) > 0
elif mask_name in adata.var:
mask_per = "v"
mask = ut.get_v_numpy(
adata, mask_name, formatter=ut.mask_description) > 0
else:
if must_exist:
raise KeyError(f"unknown mask data: {mask_name}")
continue
if mask.dtype != "bool":
raise ValueError(f"the data: {mask_name} is not a boolean mask")
if invert_mask:
mask = ~mask
if ut.logging_calc():
ut.log_calc(log_mask_name, mask)
if per is None:
per = mask_per
else:
if mask_per != per:
raise ValueError(
"mixing per-observation and per-variable masks")
if is_or:
if or_mask is None:
or_mask = mask
else:
or_mask = or_mask | mask
else:
if and_mask is None:
and_mask = mask
else:
and_mask = and_mask & mask
if and_mask is not None:
if or_mask is not None:
combined_mask = and_mask & or_mask
else:
combined_mask = and_mask
else:
if or_mask is not None:
combined_mask = or_mask
else:
raise ValueError("no masks to combine")
if invert:
combined_mask = ~combined_mask
if to is None:
ut.log_return("combined", combined_mask)
if per == "o":
return ut.to_pandas_series(combined_mask, index=adata.obs_names)
assert per == "v"
return ut.to_pandas_series(combined_mask, index=adata.var_names)
if per == "o":
ut.set_o_data(adata, to, combined_mask)
else:
ut.set_v_data(adata, to, combined_mask)
return None
def compute_inner_fold_factors(
what: Union[str, ut.Matrix] = "__x__",
*,
adata: AnnData,
gdata: AnnData,
group: Union[str, ut.Vector] = "metacell",
min_gene_inner_fold_factor: float = pr.min_gene_inner_fold_factor,
min_entry_inner_fold_factor: float = pr.min_entry_inner_fold_factor,
inner_abs_folds: float = pr.inner_abs_folds,
) -> None:
"""
Compute the inner fold factors of genes within in each metacell.
This computes, for each cell of the metacell, the same fold factors that are used to detect deviant cells (see
:py:func:`metacells.tools.deviants.find_deviant_cells`), and keeps the maximal fold for each gene in the metacell.
The result per-metacell-per-gene matrix is then made sparse by discarding too-low values (setting them to zero).
Ideally, this matrix should be "very" sparse. If it contains "too many" non-zero values, this indicates the
metacells contains "too much" variability. This may be due to actual biology (e.g. immune cells or olfactory nerves
which are all similar except for each one expressing one different gene), due to batch effects (similar cells in
distinct batches differing in some genes due to technical issues), due to low data quality (the overall noise level
is so high that this is simply the best the algorithm can do), or worse - a combination of the above.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where ``what`` is a
per-variable-per-observation matrix or the name of a per-variable-per-observation annotation containing such a
matrix.
In addition, ``gdata`` is assumed to have one observation for each group, and use the same
genes as ``adata``.
**Returns**
Sets the following in ``gdata``:
Per-Variable Per-Observation (Gene-Cell) Annotations
``inner_fold``
For each gene and group, the maximal fold factor of this gene in any cell contained in the group (unless the
value is too low to be of interest, in which case it will be zero).
**Computation Parameters**
1. For each group (metacell), for each gene, compute the gene's maximal (in all the cells of the group) fold factor
log2((actual UMIs + 1) / (expected UMIs + 1)), similarly to
:py:func:`metacells.tools.deviants.find_deviant_cells`.
2. If the maximal fold factor for a gene (across all metacells) is below ``min_gene_inner_fold_factor`` (default:
{min_gene_inner_fold_factor}), then set all the gene's fold factors to zero (too low to be of interest). If
``inner_abs_folds`` (default: {inner_abs_folds}), consider the absolute fold factors.
3. Otherwise, for any metacell whose fold factor for the gene is less than ``min_entry_inner_fold_factor`` (default:
{min_entry_inner_fold_factor}), set the fold factor to zero (too low to be of interest).
"""
assert 0 <= min_entry_inner_fold_factor <= min_gene_inner_fold_factor
cells_data = ut.get_vo_proper(adata, what, layout="row_major")
metacells_data = ut.get_vo_proper(gdata, what, layout="row_major")
group_of_cells = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
total_umis_per_cell = ut.sum_per(cells_data, per="row")
total_umis_per_metacell = ut.sum_per(metacells_data, per="row")
@ut.timed_call("compute_metacell_inner_folds")
def _compute_single_metacell_inner_folds(
metacell_index: int) -> ut.NumpyVector:
return _compute_metacell_inner_folds(
metacell_index=metacell_index,
cells_data=cells_data,
metacells_data=metacells_data,
group_of_cells=group_of_cells,
total_umis_per_cell=total_umis_per_cell,
total_umis_per_metacell=total_umis_per_metacell,
)
results = list(
ut.parallel_map(_compute_single_metacell_inner_folds, gdata.n_obs))
dense_inner_folds_by_row = np.array(results)
dense_inner_folds_by_column = ut.to_layout(dense_inner_folds_by_row,
"column_major")
if inner_abs_folds:
comparable_dense_inner_folds_by_column = np.abs(
dense_inner_folds_by_column)
else:
comparable_dense_inner_folds_by_column = dense_inner_folds_by_column
max_fold_per_gene = ut.max_per(comparable_dense_inner_folds_by_column,
per="column")
significant_genes_mask = max_fold_per_gene >= min_gene_inner_fold_factor
ut.log_calc("significant_genes_mask", significant_genes_mask)
dense_inner_folds_by_column[:, ~significant_genes_mask] = 0
dense_inner_folds_by_column[comparable_dense_inner_folds_by_column <
min_entry_inner_fold_factor] = 0
dense_inner_folds_by_row = ut.to_layout(dense_inner_folds_by_column,
layout="row_major")
sparse_inner_folds = sparse.csr_matrix(dense_inner_folds_by_row)
ut.set_vo_data(gdata, "inner_fold", sparse_inner_folds)
def compute_type_compatible_sizes(
adatas: List[AnnData],
*,
size: str = "grouped",
kind: str = "type",
) -> None:
"""
Given multiple annotated data of groups, compute a "compatible" size for each one to allow for
consistent inner normalized variance comparison.
Since the inner normalized variance quality measure is sensitive to the group (metacell) sizes,
it is useful to artificially shrink the groups so the sizes will be similar between the compared
data sets. Assuming each group (metacell) has a type annotation, for each such type, we give
each one a "compatible" size (less than or equal to its actual size) so that using this reduced
size will give us comparable measures between all the data sets.
The "compatible" sizes are chosen such that the density distributions of the sizes in all data
sets would be as similar to each other as possible.
.. note::
This is only effective if the groups are "similar" in size. Using this to compare very coarse
grouping (few thousands of cells) with fine-grained ones (few dozens of cells) will still
result in very different results.
**Input**
Several annotated ``adatas`` where each observation is a group. Should contain per-observation
``size`` annotation (default: {size}) and ``kind`` annotation (default: {kind}).
**Returns**
Sets the following in each ``adata``:
Per-Observation (group) Annotations:
``compatible_size``
The number of grouped cells in the group to use for computing excess R^2 and inner
normalized variance.
**Computation**
1. For each type, sort the groups (metacells) in increasing number of grouped observations (cells).
2. Consider the maximal quantile (rank) of the next smallest group (metacell) in each data set.
3. Compute the minimal number of grouped observations in all the metacells whose quantile is up
to this maximal quantile.
4. Use this as the "compatible" size for all these groups, and remove them from consideration.
5. Loop until all groups are assigned a "compatible" size.
"""
assert len(adatas) > 0
if len(adatas) == 1:
ut.set_o_data(
adatas[0], "compatible_size",
ut.get_o_numpy(adatas[0], size, formatter=ut.sizes_description))
return
group_sizes_of_data = [
ut.get_o_numpy(adata, size, formatter=ut.sizes_description)
for adata in adatas
]
group_types_of_data = [ut.get_o_numpy(adata, kind) for adata in adatas]
unique_types: Set[Any] = set()
for group_types in group_types_of_data:
unique_types.update(group_types)
compatible_size_of_data = [np.full(adata.n_obs, -1) for adata in adatas]
groups_count_of_data: List[int] = []
for type_index, group_type in enumerate(sorted(unique_types)):
with ut.log_step(
f"- {group_type}",
ut.progress_description(len(unique_types), type_index,
"type")):
sorted_group_indices_of_data = [
np.argsort(group_sizes)[group_types == group_type]
for group_sizes, group_types in zip(group_sizes_of_data,
group_types_of_data)
]
groups_count_of_data = [
len(sorted_group_indices)
for sorted_group_indices in sorted_group_indices_of_data
]
ut.log_calc("group_counts", groups_count_of_data)
def _for_each(value_of_data: List[T]) -> List[T]:
return [
value for groups_count, value in zip(
groups_count_of_data, value_of_data)
if groups_count > 0
]
groups_count_of_each = _for_each(groups_count_of_data)
if len(groups_count_of_each) == 0:
continue
sorted_group_indices_of_each = _for_each(
sorted_group_indices_of_data)
group_sizes_of_each = _for_each(group_sizes_of_data)
compatible_size_of_each = _for_each(compatible_size_of_data)
if len(groups_count_of_each) == 1:
compatible_size_of_each[0][
sorted_group_indices_of_each[0]] = group_sizes_of_each[0][
sorted_group_indices_of_each[0]]
group_quantile_of_each = [
(np.arange(len(sorted_group_indices)) + 1) /
len(sorted_group_indices)
for sorted_group_indices in sorted_group_indices_of_each
]
next_position_of_each = np.full(len(group_quantile_of_each), 0)
while True:
next_quantile_of_each = [
group_quantile[next_position]
for group_quantile, next_position in zip(
group_quantile_of_each, next_position_of_each)
]
next_quantile = max(next_quantile_of_each)
last_position_of_each = next_position_of_each.copy()
next_position_of_each[:] = [
np.sum(group_quantile <= next_quantile)
for group_quantile in group_quantile_of_each
]
positions_of_each = [
range(last_position, next_position)
for last_position, next_position in zip(
last_position_of_each, next_position_of_each)
]
sizes_of_each = [
group_sizes[sorted_group_indices[positions]]
for group_sizes, sorted_group_indices, positions in zip(
group_sizes_of_each, sorted_group_indices_of_each,
positions_of_each)
]
min_size_of_each = [
np.min(sizes) for sizes, positions in zip(
sizes_of_each, positions_of_each)
]
min_size = min(min_size_of_each)
for sorted_group_indices, positions, compatible_size in zip(
sorted_group_indices_of_each, positions_of_each,
compatible_size_of_each):
compatible_size[sorted_group_indices[positions]] = min_size
is_done_of_each = [
next_position == groups_count
for next_position, groups_count in zip(
next_position_of_each, groups_count_of_each)
]
if all(is_done_of_each):
break
assert not any(is_done_of_each)
for adata, compatible_size in zip(adatas, compatible_size_of_data):
assert np.min(compatible_size) > 0
ut.set_o_data(adata, "compatible_size", compatible_size)
def compute_inner_normalized_variance(
what: Union[str, ut.Matrix] = "__x__",
*,
compatible_size: Optional[str] = None,
downsample_min_samples: int = pr.downsample_min_samples,
downsample_min_cell_quantile: float = pr.downsample_min_cell_quantile,
downsample_max_cell_quantile: float = pr.downsample_max_cell_quantile,
min_gene_total: int = pr.quality_min_gene_total,
adata: AnnData,
gdata: AnnData,
group: Union[str, ut.Vector] = "metacell",
random_seed: int = pr.random_seed,
) -> None:
"""
Compute the inner normalized variance (variance / mean) for each gene in each group.
This is also known as the "index of dispersion" and can serve as a quality measure for
the groups. An ideal group would contain only cells with "the same" biological state
and all remaining inner variance would be due to technical sampling noise.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
In addition, ``gdata`` is assumed to have one observation for each group, and use the same
genes as ``adata``.
**Returns**
Sets the following in ``gdata``:
Per-Variable Per-Observation (Gene-Cell) Annotations
``inner_variance``
For each gene and group, the variance of the gene in the group.
``inner_normalized_variance``
For each gene and group, the normalized variance (variance over mean) of the
gene in the group.
**Computation Parameters**
For each group (metacell):
1. If ``compatible_size`` (default: {compatible_size}) is specified, it should be an
integer per-observation annotation of the groups, whose value is at most the
number of grouped cells in the group. Pick a random subset of the cells of
this size. If ``compatible_size`` is ``None``, use all the cells of the group.
2. Invoke :py:func:`metacells.tools.downsample.downsample_cells` to downsample the surviving
cells to the same total number of UMIs, using the ``downsample_min_samples`` (default:
{downsample_min_samples}), ``downsample_min_cell_quantile`` (default:
{downsample_min_cell_quantile}), ``downsample_max_cell_quantile`` (default:
{downsample_max_cell_quantile}) and the ``random_seed`` (default: {random_seed}).
3. Compute the normalized variance of each gene based on the downsampled data. Set the
result to ``nan`` for genes with less than ``min_gene_total`` (default: {min_gene_total}).
"""
cells_data = ut.get_vo_proper(adata, what, layout="row_major")
if compatible_size is not None:
compatible_size_of_groups: Optional[ut.NumpyVector] = ut.get_o_numpy(
gdata, compatible_size, formatter=ut.sizes_description)
else:
compatible_size_of_groups = None
group_of_cells = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
groups_count = np.max(group_of_cells) + 1
assert groups_count > 0
assert gdata.n_obs == groups_count
variance_per_gene_per_group = np.full(gdata.shape, None, dtype="float32")
normalized_variance_per_gene_per_group = np.full(gdata.shape,
None,
dtype="float32")
for group_index in range(groups_count):
with ut.log_step(
"- group",
group_index,
formatter=lambda group_index: ut.progress_description(
groups_count, group_index, "group"),
):
if compatible_size_of_groups is not None:
compatible_size_of_group = compatible_size_of_groups[
group_index]
else:
compatible_size_of_group = None
_collect_group_data(
group_index,
group_of_cells=group_of_cells,
cells_data=cells_data,
compatible_size=compatible_size_of_group,
downsample_min_samples=downsample_min_samples,
downsample_min_cell_quantile=downsample_min_cell_quantile,
downsample_max_cell_quantile=downsample_max_cell_quantile,
min_gene_total=min_gene_total,
random_seed=random_seed,
variance_per_gene_per_group=variance_per_gene_per_group,
normalized_variance_per_gene_per_group=
normalized_variance_per_gene_per_group,
)
ut.set_vo_data(gdata, "inner_variance", variance_per_gene_per_group)
ut.set_vo_data(gdata, "inner_normalized_variance",
normalized_variance_per_gene_per_group)
def find_properly_sampled_cells(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
min_cell_total: Optional[int],
max_cell_total: Optional[int],
excluded_adata: Optional[AnnData] = None,
max_excluded_genes_fraction: Optional[float],
inplace: bool = True,
) -> Optional[ut.PandasSeries]:
"""
Detect cells with a "proper" amount of ``what`` (default: {what}) data.
Due to both technical effects and natural variance between cells, the total number of UMIs
varies from cell to cell. We often would like to work on cells that contain a sufficient number
of UMIs for meaningful analysis; we sometimes also wish to exclude cells which have "too many"
UMIs.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Observation (Cell) Annotations
``properly_sampled_cell``
A boolean mask indicating whether each cell has a "proper" amount of UMIs.
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas series (indexed by the observation names).
**Computation Parameters**
1. Exclude all cells whose total data is less than the ``min_cell_total`` (no default), unless
it is ``None``.
2. Exclude all cells whose total data is more than the ``max_cell_total`` (no default), unless
it is ``None``.
3. If ``max_excluded_genes_fraction`` (no default) is not ``None``, then ``excluded_adata`` must
not be ``None`` and should contain just the excluded genes data for each cell. Exclude all
cells whose sum of the excluded data divided by the total data is more than the specified
threshold.
"""
assert (max_excluded_genes_fraction is None) == (excluded_adata is None)
total_of_cells = ut.get_o_numpy(adata, what, sum=True)
cells_mask = np.full(adata.n_obs, True, dtype="bool")
if min_cell_total is not None:
cells_mask = cells_mask & (total_of_cells >= min_cell_total)
if max_cell_total is not None:
cells_mask = cells_mask & (total_of_cells <= max_cell_total)
if excluded_adata is not None:
assert max_excluded_genes_fraction is not None
excluded_data = ut.get_vo_proper(excluded_adata, layout="row_major")
excluded_of_cells = ut.sum_per(excluded_data, per="row")
if np.min(total_of_cells) == 0:
total_of_cells = np.copy(total_of_cells)
total_of_cells[total_of_cells == 0] = 1
excluded_fraction = excluded_of_cells / total_of_cells
cells_mask = cells_mask & (excluded_fraction <=
max_excluded_genes_fraction)
if inplace:
ut.set_o_data(adata, "properly_sampled_cell", cells_mask)
return None
ut.log_return("properly_sampled_cell", cells_mask)
return ut.to_pandas_series(cells_mask, index=adata.obs_names)
def compute_deviant_fold_factors(
what: Union[str, ut.Matrix] = "__x__",
*,
adata: AnnData,
gdata: AnnData,
group: Union[str, ut.Vector] = "metacell",
similar: Union[str, ut.Vector] = "similar",
significant_gene_fold_factor: float = pr.significant_gene_fold_factor,
) -> None:
"""
Given an assignment of observations (cells) to groups (metacells) or, if an outlier, to the most
similar groups, compute for each observation and gene the fold factor relative to its group
for the purpose of detecting deviant cells.
Ideally, all grouped cells would have no genes with high enough fold factors to be considered deviants, and all
outlier cells would. In practice grouped cells might have a (few) such genes to the restriction on the fraction
of deviants.
It is important not to read too much into the results for a single cell, but looking at which genes appear for cell
populations (e.g., cells with specific metadata such as batch identification) might be instructive.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where ``what`` is a
per-variable-per-observation matrix or the name of a per-variable-per-observation annotation containing such a
matrix.
In addition, ``gdata`` is assumed to have one observation for each group, and use the same genes as ``adata``.
**Returns**
Sets the following in ``adata``:
Per-Variable Per-Observation (Gene-Cell) Annotations
``deviant_fold``
The fold factor between the cell's UMIs and the expected number of UMIs for the purpose of computing
deviant cells.
**Computation Parameters**
1. For each cell, compute the expected UMIs for each gene given the fraction of the gene in the metacells associated
with the cell (the one it is belongs to, or the most similar one for outliers). If this is less than
``significant_gene_fold_factor`` (default: {significant_gene_fold_factor}), set it to zero so the result will be
sparse.
"""
cells_data = ut.get_vo_proper(adata, what, layout="row_major")
metacells_data = ut.get_vo_proper(gdata, what, layout="row_major")
total_umis_per_cell = ut.sum_per(cells_data, per="row")
total_umis_per_metacell = ut.sum_per(metacells_data, per="row")
group_of_cells = ut.get_o_numpy(adata,
group,
formatter=ut.groups_description)
similar_of_cells = ut.get_o_numpy(adata,
similar,
formatter=ut.groups_description)
@ut.timed_call("compute_cell_deviant_certificates")
def _compute_cell_deviant_certificates(
cell_index: int) -> Tuple[ut.NumpyVector, ut.NumpyVector]:
return _compute_cell_certificates(
cell_index=cell_index,
cells_data=cells_data,
metacells_data=metacells_data,
group_of_cells=group_of_cells,
similar_of_cells=similar_of_cells,
total_umis_per_cell=total_umis_per_cell,
total_umis_per_metacell=total_umis_per_metacell,
significant_gene_fold_factor=significant_gene_fold_factor,
)
results = list(
ut.parallel_map(_compute_cell_deviant_certificates, adata.n_obs))
cell_indices = np.concatenate([
np.full(len(result[0]), cell_index, dtype="int32")
for cell_index, result in enumerate(results)
])
gene_indices = np.concatenate([result[0] for result in results])
fold_factors = np.concatenate([result[1] for result in results])
deviant_folds = sparse.csr_matrix(
(fold_factors, (cell_indices, gene_indices)), shape=adata.shape)
ut.set_vo_data(adata, "deviant_folds", deviant_folds)
def _identify_cells(
*,
adata_of_all_genes_of_all_cells: AnnData,
what: Union[str, ut.Matrix] = "__x__",
related_gene_indices_of_modules: List[List[int]],
min_cell_module_total: int,
min_cells_of_modules: int,
max_cells_of_modules: int,
rare_module_of_cells: ut.NumpyVector,
) -> None:
max_strength_of_cells = np.zeros(adata_of_all_genes_of_all_cells.n_obs)
ut.log_calc("cells for modules:")
modules_count = len(related_gene_indices_of_modules)
for module_index, related_gene_indices_of_module in enumerate(
related_gene_indices_of_modules):
if len(related_gene_indices_of_module) == 0:
continue
with ut.log_step(
"- module",
module_index,
formatter=lambda module_index: ut.progress_description(
modules_count, module_index, "module"),
):
adata_of_related_genes_of_all_cells = ut.slice(
adata_of_all_genes_of_all_cells,
name=f".module{module_index}.related_genes",
vars=related_gene_indices_of_module,
top_level=False,
)
total_related_genes_of_all_cells = ut.get_o_numpy(
adata_of_related_genes_of_all_cells, what, sum=True)
mask_of_strong_cells_of_module = total_related_genes_of_all_cells >= min_cell_module_total
median_strength_of_module = np.median(
total_related_genes_of_all_cells[
mask_of_strong_cells_of_module]) #
strong_cells_count = np.sum(mask_of_strong_cells_of_module)
if strong_cells_count > max_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"strong_cells",
ut.mask_description(mask_of_strong_cells_of_module) +
" (too many)") #
related_gene_indices_of_module.clear()
continue
if strong_cells_count < min_cells_of_modules:
if ut.logging_calc():
ut.log_calc(
"strong_cells",
ut.mask_description(mask_of_strong_cells_of_module) +
" (too few)") #
related_gene_indices_of_module.clear()
continue
ut.log_calc("strong_cells", mask_of_strong_cells_of_module)
strength_of_all_cells = total_related_genes_of_all_cells / median_strength_of_module
mask_of_strong_cells_of_module &= strength_of_all_cells >= max_strength_of_cells
max_strength_of_cells[
mask_of_strong_cells_of_module] = strength_of_all_cells[
mask_of_strong_cells_of_module]
rare_module_of_cells[mask_of_strong_cells_of_module] = module_index
def group_obs_data(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
groups: Union[str, ut.Vector],
name: Optional[str] = None,
) -> Optional[AnnData]:
"""
Compute new data which has the ``what`` (default: {what}) sum of the observations (cells) for
each group.
For example, having computed a metacell index for each cell, compute the per-metacell data
for further analysis.
If ``groups`` is a string, it is expected to be the name of a per-observation vector annotation.
Otherwise it should be a vector. The group indices should be integers, where negative values
indicate "no group" and non-negative values indicate the index of the group to which each
observation (cell) belongs to.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
An annotated data where each observation is the sum of the group of original observations
(cells). Observations with a negative group index are discarded. If all observations are
discarded, return ``None``.
The new data will contain only:
* An ``X`` member holding the summed-per-group data.
* A new ``grouped`` per-observation data which counts, for each group, the number
of grouped observations summed into it.
If ``name`` is not specified, the data will be unnamed. Otherwise, if it starts with a ``.``, it
will be appended to the current name (if any). Otherwise, ``name`` is the new name.
"""
group_of_cells = ut.get_o_numpy(adata,
groups,
formatter=ut.groups_description)
data = ut.get_vo_proper(adata, what, layout="row_major")
results = ut.sum_groups(data, group_of_cells, per="row")
if results is None:
return None
summed_data, cell_counts = results
gdata = AnnData(summed_data)
gdata.var_names = adata.var_names
ut.set_name(gdata, ut.get_name(adata))
ut.set_name(gdata, name)
ut.set_o_data(gdata,
"grouped",
cell_counts,
formatter=ut.sizes_description)
return gdata
def dissolve_metacells(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
candidates: Union[str, ut.Vector] = "candidate",
deviants: Optional[Union[str, ut.Vector]] = "cell_deviant_votes",
target_metacell_size: float = pr.target_metacell_size,
cell_sizes: Optional[Union[str, ut.Vector]] = pr.dissolve_cell_sizes,
min_metacell_cells: int = pr.dissolve_min_metacell_cells,
min_robust_size_factor: Optional[float] = pr.
dissolve_min_robust_size_factor,
min_convincing_size_factor: Optional[float] = pr.
dissolve_min_convincing_size_factor,
min_convincing_gene_fold_factor: float = pr.
dissolve_min_convincing_gene_fold_factor,
abs_folds: bool = pr.dissolve_abs_folds,
inplace: bool = True,
) -> Optional[ut.PandasFrame]:
"""
Dissolve too-small metacells based on ``what`` (default: {what}) data.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Observation (Cell) Annotations
``metacell``
The integer index of the metacell each cell belongs to. The metacells are in no
particular order. Cells with no metacell assignment are given a metacell index of
``-1``.
``dissolved``
A boolean mask of the cells which were in a dissolved metacell.
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas data frame (indexed by the observation names).
**Computation Parameters**
1. Mark all cells with non-zero ``deviants`` (default: {deviants}) as "outliers". This can be
the name of a per-observation (cell) annotation, or an explicit boolean mask of cells, or a
or ``None`` if there are no deviant cells to mark.
2. Any metacell which has less cells than the ``min_metacell_cells`` is dissolved.
3. We are trying to create metacells of size ``target_metacell_size``. Compute the sizes of the
resulting metacells by summing the ``cell_sizes`` (default: {cell_sizes}). If it is ``None``,
each has a size of one. These parameters are typically identical to these passed to
:py:func:`metacells.tools.candidates.compute_candidate_metacells`.
4. If ``min_robust_size_factor`` (default: {min_robust_size_factor}) is specified, then any
metacell whose total size is at least ``target_metacell_size * min_robust_size_factor`` is
preserved.
5. If ``min_convincing_size_factor`` (default: {min_convincing_size_factor}) is specified, then any remaining
metacells whose size is at least ``target_metacell_size * min_convincing_size_factor`` are preserved, given they
contain at least one gene whose fold factor (log2((actual + 1) / (expected + 1))) is at least
``min_convincing_gene_fold_factor`` (default: {min_convincing_gene_fold_factor}). If ``abs_folds``, consider the
absolute fold factors. That is, we only preserve these smaller metacells if there is at least one gene whose
expression is significantly different from the mean of the population.
6 . Any remaining metacell is dissolved into "outlier" cells.
"""
dissolved_of_cells = np.zeros(adata.n_obs, dtype="bool")
candidate_of_cells = ut.get_o_numpy(adata,
candidates,
formatter=ut.groups_description)
candidate_of_cells = np.copy(candidate_of_cells)
deviant_of_cells = ut.maybe_o_numpy(adata,
deviants,
formatter=ut.mask_description)
if deviant_of_cells is not None:
deviant_of_cells = deviant_of_cells > 0
cell_sizes = ut.maybe_o_numpy(adata,
cell_sizes,
formatter=ut.sizes_description)
if deviant_of_cells is not None:
candidate_of_cells[deviant_of_cells > 0] = -1
candidate_of_cells = ut.compress_indices(candidate_of_cells)
candidates_count = np.max(candidate_of_cells) + 1
data = ut.get_vo_proper(adata, what, layout="column_major")
fraction_of_genes = ut.fraction_per(data, per="column")
if min_robust_size_factor is None:
min_robust_size = None
else:
min_robust_size = target_metacell_size * min_robust_size_factor
ut.log_calc("min_robust_size", min_robust_size)
if min_convincing_size_factor is None:
min_convincing_size = None
else:
min_convincing_size = target_metacell_size * min_convincing_size_factor
ut.log_calc("min_convincing_size", min_convincing_size)
did_dissolve = False
for candidate_index in range(candidates_count):
candidate_cell_indices = np.where(
candidate_of_cells == candidate_index)[0]
if not _keep_candidate(
adata,
candidate_index,
data=data,
cell_sizes=cell_sizes,
fraction_of_genes=fraction_of_genes,
min_metacell_cells=min_metacell_cells,
min_robust_size=min_robust_size,
min_convincing_size=min_convincing_size,
min_convincing_gene_fold_factor=min_convincing_gene_fold_factor,
abs_folds=abs_folds,
candidates_count=candidates_count,
candidate_cell_indices=candidate_cell_indices,
):
dissolved_of_cells[candidate_cell_indices] = True
candidate_of_cells[candidate_cell_indices] = -1
did_dissolve = True
if did_dissolve:
metacell_of_cells = ut.compress_indices(candidate_of_cells)
else:
metacell_of_cells = candidate_of_cells
if inplace:
ut.set_o_data(adata,
"dissolved",
dissolved_of_cells,
formatter=ut.mask_description)
ut.set_o_data(adata,
"metacell",
metacell_of_cells,
formatter=ut.groups_description)
return None
ut.log_return("dissolved", dissolved_of_cells)
ut.log_return("metacell",
metacell_of_cells,
formatter=ut.groups_description)
obs_frame = ut.to_pandas_frame(index=adata.obs_names)
obs_frame["dissolved"] = dissolved_of_cells
obs_frame["metacell"] = metacell_of_cells
return obs_frame
def group_obs_annotation(
adata: AnnData,
gdata: AnnData,
*,
groups: Union[str, ut.Vector],
name: str,
formatter: Optional[Callable[[Any], Any]] = None,
method: str = "majority",
min_value_fraction: float = 0.5,
conflict: Optional[Any] = None,
inplace: bool = True,
) -> Optional[ut.PandasSeries]:
"""
Transfer per-observation data from the per-observation (cell) ``adata`` to the
per-group-of-observations (metacells) ``gdata``.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, and the
``gdata`` containing the per-metacells summed data.
**Returns**
Observations (Cell) Annotations
``<name>``
The per-group-observation annotation computed based on the per-observation annotation.
If ``inplace`` (default: {inplace}), this is written to the ``gdata``, and the function returns
``None``. Otherwise this is returned as a pandas series (indexed by the group observation
names).
**Computation Parameters**
1. Iterate on all the observations (groups, metacells) in ``gdata``.
2. Consider all the cells whose ``groups`` annotation maps them into this group.
3. Consider all the ``name`` annotation values of these cells.
4. Compute an annotation value for the whole group of cells using the ``method``. Supported
methods are:
``unique``
All the values of all the cells in the group are expected to be the same, use this
unique value for the whole groups.
``majority``
Use the most common value across all cells in the group as the value for the whole
group. If this value doesn't have at least ``min_value_fraction`` (default:
{min_value_fraction}) of the cells, use the ``conflict`` (default: {conflict}) value
instead.
"""
group_of_cells = ut.get_o_numpy(adata,
groups,
formatter=ut.groups_description)
values_of_cells = ut.get_o_numpy(adata, name, formatter=formatter)
value_of_groups = np.empty(gdata.n_obs, dtype=values_of_cells.dtype)
assert method in ("unique", "majority")
if method == "unique":
with ut.timed_step(".unique"):
value_of_groups[group_of_cells] = values_of_cells
else:
assert method == "majority"
with ut.timed_step(".majority"):
for group_index in range(gdata.n_obs):
cells_mask = group_of_cells == group_index
cells_count = np.sum(cells_mask)
assert cells_count > 0
values_of_cells_of_group = values_of_cells[cells_mask]
unique_values_of_group, unique_counts_of_group = np.unique(
values_of_cells_of_group, return_counts=True)
majority_index = np.argmax(unique_counts_of_group)
majority_count = unique_counts_of_group[majority_index]
if majority_count / cells_count < min_value_fraction:
value_of_groups[group_index] = conflict
else:
majority_value = unique_values_of_group[majority_index]
value_of_groups[group_index] = majority_value
if inplace:
ut.set_o_data(gdata, name, value_of_groups)
return None
return ut.to_pandas_series(value_of_groups, index=gdata.obs_names)
def compute_subset_distinct_genes(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
prefix: Optional[str] = None,
scale: Optional[Union[bool, str, ut.NumpyVector]],
subset: Union[str, ut.NumpyVector],
normalization: float,
) -> Optional[Tuple[ut.PandasSeries, ut.PandasSeries]]:
"""
Given a subset of the observations (cells), compute for each gene how distinct its ``what``
(default: {what}) value is in the subset compared to the overall population.
This is the area-under-curve of the receiver operating characteristic (AUROC) for the gene, that
is, the probability that a randomly selected observation (cell) in the subset will have a higher
value than a randomly selected observation (cell) outside the subset.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Variable (Gene) Annotations
``<prefix>_fold``
Store the ratio of the expression of the gene in the subset as opposed to the rest of
the population.
``<prefix>_auroc``
Store the distinctiveness of the gene in the subset as opposed to the rest of the
population.
If ``prefix`` (default: {prefix}), is specified, this is written to the data. Otherwise this is
returned as two pandas series (indexed by the gene names).
**Computation Parameters**
1. Use the ``subset`` to assign a boolean label to each observation (cell). The ``subset`` can
be a vector of integer observation names, or a boolean mask, or the string name of a
per-observation annotation containing the boolean mask.
2. If ``scale`` is ``False``, use the data as-is. If it is ``True``, divide the data by the
sum of each observation (cell). If it is a string, it should be the name of a per-observation
annotation to use. Otherwise, it should be a vector of the scale factor for each observation
(cell).
3. Compute the fold ratios using the ``normalization`` (no default!) and the AUROC for each gene,
for the scaled data based on this mask.
"""
if isinstance(subset, str):
subset = ut.get_o_numpy(adata, subset)
if subset.dtype != "bool":
mask: ut.NumpyVector = np.full(adata.n_obs, False)
mask[subset] = True
subset = mask
scale_of_cells: Optional[ut.NumpyVector] = None
if not isinstance(scale, bool):
scale_of_cells = ut.maybe_o_numpy(adata,
scale,
formatter=ut.sizes_description)
elif scale:
scale_of_cells = ut.get_o_numpy(adata, what, sum=True)
else:
scale_of_cells = None
matrix = ut.get_vo_proper(adata, what, layout="column_major").transpose()
fold_of_genes, auroc_of_genes = ut.matrix_rows_folds_and_aurocs(
matrix,
columns_subset=subset,
columns_scale=scale_of_cells,
normalization=normalization)
if prefix is not None:
ut.set_v_data(adata, f"{prefix}_auroc", auroc_of_genes)
ut.set_v_data(adata, f"{prefix}_fold", fold_of_genes)
return None
return (
ut.to_pandas_series(fold_of_genes, index=adata.var_names),
ut.to_pandas_series(auroc_of_genes, index=adata.var_names),
)
def compute_groups_self_consistency(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
group: str = "metacell",
genes_mask: Optional[ut.NumpyVector] = None,
self_similarity_log_data: bool = pr.self_similarity_log_data,
self_similarity_value_normalization: float = pr.
self_similarity_value_normalization,
self_similarity_method: str = pr.self_similarity_method,
reproducible: bool = pr.reproducible,
logistics_location: float = pr.logistics_location,
logistics_slope: float = pr.logistics_slope,
) -> ut.NumpyVector:
"""
Compute the self consistency (similarity between two halves) of some groups.
**Input**
The input annotated ``adata`` is expected to contain a per-observation annotation named
``group`` (default: {group}) which identifies the group (metacells) each observation (cell)
belongs to, and ``half_<group>`` which identifies the half-group each observation belongs
to (e.g. as computed by :py:func:`split_groups`). Specifically, the indices of the halves
of group index ``i`` are ``i`` and ``i + groups_count``.
**Returns**
A Numpy vector holding, for each group, the similarity between its two halves.
**Computation Parameters**
1. For each group, compute the sum of values in each half and normalize it to fractions (sum of 1).
2. If ``genes_mask`` is specified, select only the genes specified in it. Note the sum of the
fractions of the genes of each group in the result will be less than or equal to 1.
3. If ``self_similarity_log_data`` (default: {self_similarity_log_data}), log2 the values using
``self_similarity_value_normalization`` (default: {self_similarity_value_normalization}).
4. Compute the ``self_similarity_method`` (default: {self_similarity_method}) between the two
halves. If this is the ``logistics`` similarity, then this will use ``logistics_location``
(default: {logistics_location}) and ``logistics_slope`` (default: {logistics_slope}). If this
is ``pearson``, and if ``reproducible`` (default: {reproducible}) is ``True``, a slower
(still parallel) but reproducible algorithm will be used to compute Pearson correlations.
"""
hdata = tl.group_obs_data(adata,
what,
groups=f"half_{group}",
name=".halves")
assert hdata is not None
sum_of_halves = ut.get_o_numpy(hdata, f"{what}|sum")
halves_values = ut.to_numpy_matrix(
ut.get_vo_proper(hdata, what, layout="row_major"))
halves_data = ut.mustbe_numpy_matrix(
ut.scale_by(halves_values, sum_of_halves, by="row"))
if self_similarity_value_normalization > 0:
halves_data += self_similarity_value_normalization
if self_similarity_log_data:
halves_data = ut.log_data(halves_data, base=2)
if genes_mask is not None:
halves_data = halves_data[:, genes_mask]
assert hdata.n_obs % 2 == 0
groups_count = hdata.n_obs // 2
low_half_indices = np.arange(groups_count)
high_half_indices = low_half_indices + groups_count
low_half_data = halves_data[low_half_indices, :]
high_half_data = halves_data[high_half_indices, :]
assert self_similarity_method in ("logistics", "pearson")
if self_similarity_method == "logistics":
similarity = ut.pairs_logistics_rows(low_half_data,
high_half_data,
location=logistics_location,
slope=logistics_slope)
similarity *= -1
similarity += 1
else:
similarity = ut.pairs_corrcoef_rows(low_half_data,
high_half_data,
reproducible=reproducible)
return similarity
def renormalize_query_by_atlas( # pylint: disable=too-many-statements,too-many-branches
what: str = "__x__",
*,
adata: AnnData,
qdata: AnnData,
var_annotations: Dict[str, Any],
layers: Dict[str, Any],
varp_annotations: Dict[str, Any],
) -> Optional[AnnData]:
"""
Add an ``ATLASNORM`` pseudo-gene to query metacells data to compensate for the query having filtered out many genes.
This renormalizes the gene fractions in the query to fit the atlas in case the query has aggressive filtered a
significant amount of genes.
**Input**
Annotated query ``qdata`` and atlas ``adata``, where the observations are cells and the variables are genes, where
``X`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation annotation containing
such a matrix.
**Returns**
None if no normalization is needed (or possible). Otherwise, a copy of the query metacells data, with an additional
variable (gene) called ``ATLASNORM`` to the query data, such that the total number of UMIs for each query metacells
is as expected given the total number of UMIs of the genes common to the query and the atlas. This is skipped if the
query and the atlas have exactly the same list of genes, or if if the query already contains a high number of genes
missing from the atlas so that the total number of UMIs for the query metacells is already at least the expected
based on the common genes.
**Computation Parameters**
1. Computes how many UMIs should be added to each query metacell so that its (total UMIs / total common gene UMIs)
would be the same as the (total atlas UMIs / total atlas common UMIs). If this is zero (or negative), stop.
2. Add an ``ATLASNORM`` pseudo-gene to the query with the above amount of UMIs. For each per-variable (gene)
observation, add the value specified in ``var_annotations``, whose list of keys must cover the set of
per-variable annotations in the query data. For each per-observation-per-variable layer, add the value specified
in ``layers``, whose list of keys must cover the existing layers. For each per-variable-per-variable annotation,
add the value specified in ``varp_annotations``.
"""
for name in qdata.var.keys():
if "|" not in name and name not in var_annotations.keys():
raise RuntimeError(f"missing default value for variable annotation {name}")
for name in qdata.layers.keys():
if name not in layers.keys():
raise RuntimeError(f"missing default value for layer {name}")
for name in qdata.varp.keys():
if name not in varp_annotations.keys():
raise RuntimeError(f"missing default value for variable-variable {name}")
if list(qdata.var_names) == list(adata.var_names):
return None
query_genes_list = list(qdata.var_names)
atlas_genes_list = list(adata.var_names)
common_genes_list = list(sorted(set(qdata.var_names) & set(adata.var_names)))
query_gene_indices = np.array([query_genes_list.index(gene) for gene in common_genes_list])
atlas_gene_indices = np.array([atlas_genes_list.index(gene) for gene in common_genes_list])
common_qdata = ut.slice(qdata, name=".common", vars=query_gene_indices, track_var="full_index")
common_adata = ut.slice(adata, name=".common", vars=atlas_gene_indices, track_var="full_index")
assert list(common_qdata.var_names) == list(common_adata.var_names)
atlas_total_umis_per_metacell = ut.get_o_numpy(adata, what, sum=True)
atlas_common_umis_per_metacell = ut.get_o_numpy(common_adata, what, sum=True)
atlas_total_umis = np.sum(atlas_total_umis_per_metacell)
atlas_common_umis = np.sum(atlas_common_umis_per_metacell)
atlas_disjoint_umis_fraction = atlas_total_umis / atlas_common_umis - 1.0
ut.log_calc("atlas_total_umis", atlas_total_umis)
ut.log_calc("atlas_common_umis", atlas_common_umis)
ut.log_calc("atlas_disjoint_umis_fraction", atlas_disjoint_umis_fraction)
query_total_umis_per_metacell = ut.get_o_numpy(qdata, what, sum=True)
query_common_umis_per_metacell = ut.get_o_numpy(common_qdata, what, sum=True)
query_total_umis = np.sum(query_total_umis_per_metacell)
query_common_umis = np.sum(query_common_umis_per_metacell)
query_disjoint_umis_fraction = query_total_umis / query_common_umis - 1.0
ut.log_calc("query_total_umis", query_total_umis)
ut.log_calc("query_common_umis", query_common_umis)
ut.log_calc("query_disjoint_umis_fraction", query_disjoint_umis_fraction)
if query_disjoint_umis_fraction >= atlas_disjoint_umis_fraction:
return None
query_normalization_umis_fraction = atlas_disjoint_umis_fraction - query_disjoint_umis_fraction
ut.log_calc("query_normalization_umis_fraction", query_normalization_umis_fraction)
query_normalization_umis_per_metacell = query_common_umis_per_metacell * query_normalization_umis_fraction
_proper, dense, compressed = ut.to_proper_matrices(qdata.X)
if dense is None:
assert compressed is not None
dense = ut.to_numpy_matrix(compressed)
added = np.concatenate([dense, query_normalization_umis_per_metacell[:, np.newaxis]], axis=1)
if compressed is not None:
added = sp.csr_matrix(added)
assert added.shape[0] == qdata.shape[0]
assert added.shape[1] == qdata.shape[1] + 1
ndata = AnnData(added)
ndata.obs_names = qdata.obs_names
var_names = list(qdata.var_names)
var_names.append("ATLASNORM")
ndata.var_names = var_names
for name, value in qdata.uns.items():
ut.set_m_data(ndata, name, value)
for name, value in qdata.obs.items():
ut.set_o_data(ndata, name, value)
for name, value in qdata.obsp.items():
ut.set_oo_data(ndata, name, value)
for name in qdata.var.keys():
if "|" in name:
continue
value = ut.get_v_numpy(qdata, name)
value = np.append(value, [var_annotations[name]])
ut.set_v_data(ndata, name, value)
for name in qdata.layers.keys():
data = ut.get_vo_proper(qdata, name)
_proper, dense, compressed = ut.to_proper_matrices(data)
if dense is None:
assert compressed is not None
dense = ut.to_numpy_matrix(compressed)
values = np.full(qdata.n_obs, layers[name], dtype=dense.dtype)
added = np.concatenate([dense, values[:, np.newaxis]], axis=1)
if compressed is not None:
added = sp.csr_matrix(added)
ut.set_vo_data(ndata, name, added)
for name in qdata.varp.keys():
data = ut.get_vv_proper(qdata, name)
_proper, dense, compressed = ut.to_proper_matrices(data)
if dense is None:
assert compressed is not None
dense = ut.to_numpy_matrix(compressed)
values = np.full(qdata.n_vars, varp_annotations[name], dtype=dense.dtype)
added = np.concatenate([dense, values[:, np.newaxis]], axis=1)
values = np.full(qdata.n_vars + 1, varp_annotations[name], dtype=dense.dtype)
added = np.concatenate([added, values[:, np.newaxis]], axis=0)
if compressed is not None:
added = sp.csr_matrix(added)
ut.set_vv_data(ndata, name, added)
return ndata
def split_groups(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
group: str = "metacell",
feature_downsample_min_samples: int = pr.feature_downsample_min_samples,
feature_downsample_min_cell_quantile: float = pr.
feature_downsample_min_cell_quantile,
feature_downsample_max_cell_quantile: float = pr.
feature_downsample_max_cell_quantile,
feature_min_gene_total: Optional[int] = None,
feature_min_gene_top3: Optional[int] = None,
feature_min_gene_relative_variance: Optional[float] = pr.
feature_min_gene_relative_variance,
forbidden_gene_names: Optional[Collection[str]] = None,
forbidden_gene_patterns: Optional[Collection[Union[str, Pattern]]] = None,
cells_similarity_value_normalization: float = pr.
cells_similarity_value_normalization,
cells_similarity_log_data: bool = pr.cells_similarity_log_data,
cells_similarity_method: str = pr.cells_similarity_method,
max_cell_size: Optional[float] = pr.max_cell_size,
max_cell_size_factor: Optional[float] = pr.max_cell_size_factor,
knn_balanced_ranks_factor: float = pr.knn_balanced_ranks_factor,
knn_incoming_degree_factor: float = pr.knn_incoming_degree_factor,
knn_outgoing_degree_factor: float = pr.knn_outgoing_degree_factor,
min_seed_size_quantile: float = pr.min_seed_size_quantile,
max_seed_size_quantile: float = pr.max_seed_size_quantile,
candidates_cooldown_pass: float = pr.cooldown_pass,
candidates_cooldown_node: float = pr.cooldown_node,
random_seed: int = pr.random_seed,
) -> None:
"""
Split each metacell into two parts using ``what`` (default: {what}) data.
This creates a new partition of cells into half-metacells, which can used to
:py:func:`compute_groups_self_consistency`.
**Input**
The input annotated ``adata`` is expected to contain a per-observation annotation named
``group`` (default: {group}) which identifies the group (metacells) each observation (cell)
belongs to.
**Returns**
Sets the following annotations in ``adata``:
Observation (Cell) Annotations
``half_<group>``
The index of the half-group each cell belongs to. This is ``-1`` for ungrouped cells.
Indices 0 to the number of groups are the first (low) halves; from the number of groups
to twice that are the second (low) halves.
**Computation Parameters**
1. For each group (metacell), invoke
:py:func:`metacells.pipeline.direct.compute_direct_metacells` on the observations (cells)
included in the group, forcing the creation of two half-groups that cover all the group's
cells. The parameters are passed to this call as-is, setting ``must_complete_cover`` to
``True`` (that is, disabling outliers detection), and disabling restrictions on the
half-group sizes.
"""
group_of_cells = ut.get_o_numpy(adata, group)
groups_count = np.max(group_of_cells) + 1
half_groups_of_cells = np.full(adata.n_obs, -1, dtype="int32")
@ut.timed_call("split_group")
def split_group(group_index: int) -> Tuple[ut.NumpyVector, ut.NumpyVector]:
group_cells_mask = group_of_cells == group_index
assert np.any(group_cells_mask)
name = f".{group}-{group_index}/{groups_count}"
gdata = ut.slice(adata,
name=name,
top_level=False,
obs=group_cells_mask,
track_obs="complete_cell_index")
target_metacell_size = (gdata.n_obs + 1) // 2
compute_direct_metacells(
gdata,
what,
feature_downsample_min_samples=feature_downsample_min_samples,
feature_downsample_min_cell_quantile=
feature_downsample_min_cell_quantile,
feature_downsample_max_cell_quantile=
feature_downsample_max_cell_quantile,
feature_min_gene_total=feature_min_gene_total,
feature_min_gene_top3=feature_min_gene_top3,
feature_min_gene_relative_variance=
feature_min_gene_relative_variance,
forbidden_gene_names=forbidden_gene_names,
forbidden_gene_patterns=forbidden_gene_patterns,
cells_similarity_value_normalization=
cells_similarity_value_normalization,
cells_similarity_log_data=cells_similarity_log_data,
cells_similarity_method=cells_similarity_method,
target_metacell_size=target_metacell_size,
max_cell_size=max_cell_size,
max_cell_size_factor=max_cell_size_factor,
cell_sizes=None,
knn_k=target_metacell_size,
min_knn_k=target_metacell_size,
knn_balanced_ranks_factor=knn_balanced_ranks_factor,
knn_incoming_degree_factor=knn_incoming_degree_factor,
knn_outgoing_degree_factor=knn_outgoing_degree_factor,
min_seed_size_quantile=min_seed_size_quantile,
max_seed_size_quantile=max_seed_size_quantile,
candidates_cooldown_pass=candidates_cooldown_pass,
candidates_cooldown_node=candidates_cooldown_node,
candidates_min_split_size_factor=None,
candidates_max_merge_size_factor=None,
candidates_min_metacell_cells=1,
must_complete_cover=True,
random_seed=random_seed,
)
direct_groups = ut.get_o_numpy(gdata, "metacell")
zero_count = np.sum(direct_groups == 0)
one_count = np.sum(direct_groups == 1)
ut.log_calc(f"group: {group_index} size: {len(direct_groups)} "
f"split into: {zero_count} + {one_count}")
assert zero_count + one_count == len(direct_groups)
assert zero_count > 0
assert one_count > 0
return (group_cells_mask, group_index + groups_count * direct_groups)
for (group_cells_mask,
group_cells_halves) in ut.parallel_map(split_group, groups_count):
half_groups_of_cells[group_cells_mask] = group_cells_halves
ut.set_o_data(adata,
f"half_{group}",
half_groups_of_cells,
formatter=ut.groups_description)
def downsample_cells(
adata: AnnData,
what: Union[str, ut.Matrix] = "__x__",
*,
downsample_min_cell_quantile: float = pr.downsample_min_cell_quantile,
downsample_min_samples: float = pr.downsample_min_samples,
downsample_max_cell_quantile: float = pr.downsample_max_cell_quantile,
random_seed: int = pr.random_seed,
inplace: bool = True,
) -> Optional[ut.PandasFrame]:
"""
Downsample the values of ``what`` (default: {what}) data.
Downsampling is an effective way to get the same number of samples in multiple cells
(that is, the same number of total UMIs in multiple cells), and serves as an alternative to
normalization (e.g., working with UMI fractions instead of raw UMI counts).
Downsampling is especially important when computing correlations between cells. When there is
high variance between the total UMI count in different cells, then normalization will return
higher correlation values between cells with a higher total UMI count, which will result in an
inflated estimation of their similarity to other cells. Downsampling avoids this effect.
**Input**
Annotated ``adata``, where the observations are cells and the variables are genes, where
``what`` is a per-variable-per-observation matrix or the name of a per-variable-per-observation
annotation containing such a matrix.
**Returns**
Variable-Observation (Gene-Cell) Annotations
``downsampled``
The downsampled data where the total number of samples in each cell is at most
``samples``.
If ``inplace`` (default: {inplace}), this is written to the data, and the function returns
``None``. Otherwise this is returned as a pandas data frame (indexed by the cell and gene
names).
**Computation Parameters**
1. Compute the total samples in each cell.
2. Decide on the value to downsample to. We would like all cells to end up with at least some
reasonable number of samples (total UMIs) ``downsample_min_samples`` (default:
{downsample_min_samples}). We'd also like all (most) cells to end up with the highest
reasonable downsampled total number of samples, so if possible we increase the number of
samples, as long as at most ``downsample_min_cell_quantile`` (default:
{downsample_min_cell_quantile}) cells will have lower number of samples. We'd also like all
(most) cells to end up with the same downsampled total number of samples, so if we have to we
decrease the number of samples to ensure at most ``downsample_max_cell_quantile`` (default:
{downsample_max_cell_quantile}) cells will have a lower number of samples.
3. Downsample each cell so that it has at most the selected number of samples. Use the
``random_seed`` to allow making this replicable.
"""
total_per_cell = ut.get_o_numpy(adata, what, sum=True)
samples = int(
round(
min(
max(downsample_min_samples,
np.quantile(total_per_cell, downsample_min_cell_quantile)),
np.quantile(total_per_cell, downsample_max_cell_quantile),
)))
ut.log_calc("samples", samples)
data = ut.get_vo_proper(adata, what, layout="row_major")
assert ut.shaped_dtype(data) == "float32"
downsampled = ut.downsample_matrix(data,
per="row",
samples=samples,
random_seed=random_seed)
if inplace:
ut.set_vo_data(adata, "downsampled", downsampled)
return None
return ut.to_pandas_frame(downsampled,
index=adata.obs_names,
columns=adata.var_names)
def _apply_annotations( # pylint: disable=too-many-branches
adata: AnnData,
sdata: AnnData,
per: str,
annotations: Dict[str, DefaultValues],
indices: Union[str, ut.Vector],
) -> None:
full_name = ut.get_name(adata)
slice_name = ut.get_name(sdata)
assert per in ("o", "v")
if per == "o":
full_data = adata.obs
full_size = adata.n_obs
slice_data = sdata.obs
slice_size = sdata.n_obs
full_indices = ut.get_o_numpy(sdata, indices)
else:
full_data = adata.var
full_size = adata.n_vars
slice_data = sdata.var
slice_size = sdata.n_vars
full_indices = ut.get_v_numpy(sdata, indices)
for name, default_values in annotations.items():
slice_value = slice_data.get(name)
if slice_value is not None:
formatter: Optional[Callable[[Any], str]] = None
else:
if default_values.slice == Skip or isinstance(
default_values.slice, Skip):
continue
if default_values.slice == Raise or isinstance(
default_values.slice, Raise):
if slice_name is None:
raise KeyError(f"unknown slice data name: {name}")
raise KeyError(
f"unknown slice data: {slice_name} name: {name}")
slice_value = default_values.slice
def formatter(_: Any) -> str:
# pylint: disable=cell-var-from-loop
return f"{slice_size} <- {slice_value}"
# pylint: enable=cell-var-from-loop
full_value = full_data.get(name)
if full_value is not None:
ut.unfreeze(full_value)
else:
if default_values.full == Skip or isinstance(
default_values.full, Skip):
continue
if default_values.full == Raise or isinstance(
default_values.full, Raise):
if full_name is None:
raise KeyError(f"unknown full data name: {name}")
raise KeyError(f"unknown full data: {full_name} name: {name}")
if default_values.full is None:
full_value = np.full(full_size, None, dtype="float32")
else:
full_value = np.full(full_size, default_values.full)
full_value[full_indices] = slice_value
if per == "o":
ut.set_o_data(adata, name, full_value, formatter=formatter)
else:
ut.set_v_data(adata, name, full_value, formatter=formatter)
