def net_fle(
data: MultimodalData,
file_name: str = None,
n_jobs: int = -1,
rep: str = "diffmap",
K: int = 50,
full_speed: bool = False,
target_change_per_node: float = 2.0,
target_steps: int = 5000,
is3d: bool = False,
memory: int = 8,
random_state: int = 0,
select_frac: float = 0.1,
select_K: int = 25,
select_alpha: float = 1.0,
net_alpha: float = 0.1,
polish_target_steps: int = 1500,
out_basis: str = "net_fle",
) -> None:
"""Construct Net-Force-directed (FLE) graph.
Net-FLE is an approximated FLE graph using Deep Learning model to improve the speed.
In specific, the deep model used is MLPRegressor_, the *scikit-learn* implementation of Multi-layer Perceptron regressor.
See [Li20]_ for details.
.. _MLPRegressor: https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPRegressor.html
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
file_name: ``str``, optional, default: ``None``
Temporary file to store the coordinates as the input to forceatlas2. If ``None``, use ``tempfile.mkstemp`` to generate file name.
n_jobs: ``int``, optional, default: ``-1``
Number of threads to use. If ``-1``, use all available threads.
rep: ``str``, optional, default: ``"diffmap"``
Representation of data used for the calculation. By default, use Diffusion Map coordinates. If ``None``, use the count matrix ``data.X``.
K: ``int``, optional, default: ``50``
Number of nearest neighbors to be considered during the computation.
full_speed: ``bool``, optional, default: ``False``
* If ``True``, use multiple threads in constructing ``hnsw`` index. However, the kNN results are not reproducible.
* Otherwise, use only one thread to make sure results are reproducible.
target_change_per_node: ``float``, optional, default: ``2.0``
Target change per node to stop ForceAtlas2.
target_steps: ``int``, optional, default: ``5000``
Maximum number of iterations before stopping the ForceAtlas2 algorithm.
is3d: ``bool``, optional, default: ``False``
If ``True``, calculate 3D force-directed layout.
memory: ``int``, optional, default: ``8``
Memory size in GB for the Java FA2 component. By default, use 8GB memory.
random_state: ``int``, optional, default: ``0``
Random seed set for reproducing results.
select_frac: ``float``, optional, default: ``0.1``
Down sampling fraction on the cells.
select_K: ``int``, optional, default: ``25``
Number of neighbors to be used to estimate local density for each data point for down sampling.
select_alpha: ``float``, optional, default: ``1.0``
Weight the down sample to be proportional to ``radius ** select_alpha``.
net_alpha: ``float``, optional, default: ``0.1``
L2 penalty (regularization term) parameter of the deep regressor.
polish_target_steps: ``int``, optional, default: ``1500``
After running the deep regressor to predict new coordinate, Number of ForceAtlas2 iterations.
out_basis: ``str``, optional, default: ``"net_fle"``
Key name for calculated FLE coordinates to store.
Returns
-------
``None``
Update ``data.obsm``:
* ``data.obsm['X_' + out_basis]``: Net FLE coordinates of the data.
Update ``data.obs``:
* ``data.obs['ds_selected']``: Boolean array to indicate which cells are selected during the down sampling phase.
Examples
--------
>>> pg.net_fle(data)
"""
if file_name is None:
if file_name is None:
import tempfile
_, file_name = tempfile.mkstemp()
n_jobs = effective_n_jobs(n_jobs)
rep = update_rep(rep)
if ("W_" + rep) not in data.uns:
neighbors(
data,
K=K,
rep=rep,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed,
)
indices_key = rep + "_knn_indices"
distances_key = rep + "_knn_distances"
if not knn_is_cached(data, indices_key, distances_key, select_K):
raise ValueError("Please run neighbors first!")
selected = select_cells(
data.uns[distances_key],
select_frac,
K=select_K,
alpha=select_alpha,
random_state=random_state,
)
X_full = X_from_rep(data, rep)
X = X_full[selected, :]
ds_indices_key = "ds_" + rep + "_knn_indices"
ds_distances_key = "ds_" + rep + "_knn_distances"
indices, distances = calculate_nearest_neighbors(X,
K=K,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed)
data.uns[ds_indices_key] = indices
data.uns[ds_distances_key] = distances
W = calculate_affinity_matrix(indices, distances)
X_fle = calc_force_directed_layout(
W,
file_name + ".small",
n_jobs,
target_change_per_node,
target_steps,
is3d,
memory,
random_state,
)
data.uns["X_" + out_basis + "_small"] = X_fle
data.obs["ds_diffmap_selected"] = selected
n_components = 2 if not is3d else 3
Y_init = np.zeros((data.shape[0], n_components), dtype=np.float64)
Y_init[selected, :] = X_fle
Y_init[~selected, :] = net_train_and_predict(X,
X_fle,
X_full[~selected, :],
net_alpha,
random_state,
verbose=True)
data.obsm["X_" + out_basis + "_pred"] = Y_init
data.obsm["X_" + out_basis] = calc_force_directed_layout(
W_from_rep(data, rep),
file_name,
n_jobs,
target_change_per_node,
polish_target_steps,
is3d,
memory,
random_state,
init=Y_init,
)
def net_umap(
data: MultimodalData,
rep: str = "pca",
n_jobs: int = -1,
n_components: int = 2,
n_neighbors: int = 15,
min_dist: float = 0.5,
spread: float = 1.0,
random_state: int = 0,
select_frac: float = 0.1,
select_K: int = 25,
select_alpha: float = 1.0,
full_speed: bool = False,
net_alpha: float = 0.1,
polish_learning_rate: float = 10.0,
polish_n_epochs: int = 30,
out_basis: str = "net_umap",
) -> None:
"""Calculate Net-UMAP embedding of cells.
Net-UMAP is an approximated UMAP embedding using Deep Learning model to improve the speed.
In specific, the deep model used is MLPRegressor_, the *scikit-learn* implementation of Multi-layer Perceptron regressor.
See [Li20]_ for details.
.. _MLPRegressor: https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPRegressor.html
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
rep: ``str``, optional, default: ``"pca"``
Representation of data used for the calculation. By default, use PCA coordinates. If ``None``, use the count matrix ``data.X``.
n_components: ``int``, optional, default: ``2``
Dimension of calculated UMAP coordinates. By default, generate 2-dimensional data for 2D visualization.
n_neighbors: ``int``, optional, default: ``15``
Number of nearest neighbors considered during the computation.
min_dist: ``float``, optional, default: ``0.5``
The effective minimum distance between embedded data points.
spread: ``float``, optional, default: ``1.0``
The effective scale of embedded data points.
random_state: ``int``, optional, default: ``0``
Random seed set for reproducing results.
select_frac: ``float``, optional, default: ``0.1``
Down sampling fraction on the cells.
select_K: ``int``, optional, default: ``25``
Number of neighbors to be used to estimate local density for each data point for down sampling.
select_alpha: ``float``, optional, default: ``1.0``
Weight the down sample to be proportional to ``radius ** select_alpha``.
full_speed: ``bool``, optional, default: ``False``
* If ``True``, use multiple threads in constructing ``hnsw`` index. However, the kNN results are not reproducible.
* Otherwise, use only one thread to make sure results are reproducible.
net_alpha: ``float``, optional, default: ``0.1``
L2 penalty (regularization term) parameter of the deep regressor.
polish_learning_frac: ``float``, optional, default: ``10.0``
After running the deep regressor to predict new coordinates, use ``polish_learning_frac`` * ``n_obs`` as the learning rate to polish the coordinates.
polish_n_iter: ``int``, optional, default: ``30``
Number of iterations for polishing UMAP run.
out_basis: ``str``, optional, default: ``"net_umap"``
Key name for calculated UMAP coordinates to store.
Returns
-------
``None``
Update ``data.obsm``:
* ``data.obsm['X_' + out_basis]``: Net UMAP coordinates of the data.
Update ``data.obs``:
* ``data.obs['ds_selected']``: Boolean array to indicate which cells are selected during the down sampling phase.
Examples
--------
>>> pg.net_umap(data)
"""
rep = update_rep(rep)
indices_key = rep + "_knn_indices"
distances_key = rep + "_knn_distances"
if not knn_is_cached(data, indices_key, distances_key, select_K):
raise ValueError("Please run neighbors first!")
n_jobs = effective_n_jobs(n_jobs)
selected = select_cells(
data.uns[distances_key],
select_frac,
K=select_K,
alpha=select_alpha,
random_state=random_state,
)
X_full = X_from_rep(data, rep)
X = X_full[selected, :]
ds_indices_key = "ds_" + rep + "_knn_indices" # ds refers to down-sampling
ds_distances_key = "ds_" + rep + "_knn_distances"
indices, distances = calculate_nearest_neighbors(
X,
K=n_neighbors,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed,
)
data.uns[ds_indices_key] = indices
data.uns[ds_distances_key] = distances
knn_indices = np.insert(data.uns[ds_indices_key][:, 0:n_neighbors - 1],
0,
range(X.shape[0]),
axis=1)
knn_dists = np.insert(data.uns[ds_distances_key][:, 0:n_neighbors - 1],
0,
0.0,
axis=1)
X_umap = calc_umap(
X,
n_components,
n_neighbors,
min_dist,
spread,
random_state,
knn_indices=knn_indices,
knn_dists=knn_dists,
)
data.uns["X_" + out_basis + "_small"] = X_umap
data.obs["ds_selected"] = selected
Y_init = np.zeros((data.shape[0], n_components), dtype=np.float64)
Y_init[selected, :] = X_umap
Y_init[~selected, :] = net_train_and_predict(X,
X_umap,
X_full[~selected, :],
net_alpha,
random_state,
verbose=True)
data.obsm["X_" + out_basis + "_pred"] = Y_init
knn_indices = np.insert(data.uns[indices_key][:, 0:n_neighbors - 1],
0,
range(data.shape[0]),
axis=1)
knn_dists = np.insert(data.uns[distances_key][:, 0:n_neighbors - 1],
0,
0.0,
axis=1)
data.obsm["X_" + out_basis] = calc_umap(
X_full,
n_components,
n_neighbors,
min_dist,
spread,
random_state,
init=Y_init,
n_epochs=polish_n_epochs,
learning_rate=polish_learning_rate,
knn_indices=knn_indices,
knn_dists=knn_dists,
)
def integrative_nmf(
data: Union[MultimodalData, UnimodalData],
batch: str = "Channel",
n_components: int = 20,
features: str = "highly_variable_features",
space: str = "log",
algo: str = "halsvar",
mode: str = "online",
tol: float = 1e-4,
use_gpu: bool = False,
lam: float = 5.0,
fp_precision: str = "float",
n_jobs: int = -1,
random_state: int = 0,
quantile_norm: bool = True,
) -> str:
"""Perform Integrative Nonnegative Matrix Factorization (iNMF) [Yang16]_ for data integration.
The calculation uses `nmf-torch <https://github.com/lilab-bcb/nmf-torch>`_ .
This function assumes that cells in each batch are adjacent to each other.
In addition, it will scale each batch with L2 norm separately. The resulting Hs will also be scaled with L2 norm.
If ``quantile_norm=True``, quantile normalization will be additionally performed.
See [Welch19]_ and [Gao21]_ for preprocessing and normalization details.
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
batch: ``str``, optional, default: ``"Channel"``.
Which attribute in data.obs field represents batches, default is "Channel".
n_components: ``int``, optional, default: ``50``.
Number of Principal Components to get.
features: ``str``, optional, default: ``"highly_variable_features"``.
Keyword in ``data.var`` to specify features used for integrative_nmf.
space: ``str``, optional, default: ``log``.
Choose from ``log`` and ``expression``. ``log`` works on log-transformed expression space; ``expression`` works on the original expression space (normalized by total UMIs).
algo: ``str``, optional, default: ``halsvar``
Choose from ``mu`` (Multiplicative Update), ``halsvar`` (HALS variant that mimic bpp but faster) and ``bpp`` (alternative non-negative least squares with Block Principal Pivoting method).
mode: ``str``, optional, default: ``online``
Learning mode. Choose from ``batch`` and ``online``. Notice that ``online`` only works when ``beta=2.0``. For other beta loss, it switches back to ``batch`` method.
tol: ``float``, optional, default: ``1e-4``
The toleration used for convergence check.
use_gpu: ``bool``, optional, default: ``False``
If ``True``, use GPU if available. Otherwise, use CPU only.
lam: ``float``, optional, default: ``5.0``
The coefficient for regularization terms. If ``0``, then no regularization will be performed.
fp_precision: ``str``, optional, default: ``float``
The numeric precision on the results. Choose from ``float`` and ``double``.
n_jobs : `int`, optional (default: -1)
Number of threads to use. -1 refers to using all physical CPU cores.
random_state: ``int``, optional, default: ``0``.
Random seed to be set for reproducing result.
quantile_norm: ``bool``, optioanl, default: ``True``.
Perform quantile normalization as described in Gao et al. Nature Biotech 2021. Cluster refinement K=20; min_cells=20; quantiles = 50.
Returns
-------
out_rep: ``str``
The keyword in ``data.obsm`` referring to the embedding calculated by integrative NMF algorithm. out_rep is always equal to "inmf"
Update ``data.obsm``:
* ``data.obsm["X_inmf"]``: Scaled and possibly quantile normalized iNMF coordinates.
* ``data.obsm["H"]``: The concatenation of coordinate factor matrices of shape ``(n_cells, n_components)``.
Update ``data.uns``:
* ``data.uns["W"]``: The feature factor matrix of shape ``(n_HVFs, n_components)``.
* ``data.uns["V"]``: The batch specific feature factor matrices as one tensor of shape ``(n_batches, n_components, n_HVFs)``.
* ``data.uns["inmf_err"]``: The iNMF loss.
* ``data.uns["inmf_features"]``: Record the features used to perform iNMF analysis.
Examples
--------
>>> pg.integrative_nmf(data)
"""
if not check_batch_key(data, batch, "Cannot apply integrative_nmf!"):
return "pca"
Xs = _select_and_scale_features(data,
features=features,
space=space,
batch=batch)
try:
from nmf import integrative_nmf
except ImportError as e:
import sys
logger.error(f"{e}\nNeed NMF-Torch! Try 'pip install nmf-torch'.")
sys.exit(-1)
n_jobs = eff_n_jobs(n_jobs)
Hs, W, Vs, err = integrative_nmf(
Xs,
n_components=n_components,
algo=algo,
mode=mode,
tol=tol,
n_jobs=n_jobs,
random_state=random_state,
use_gpu=use_gpu,
lam=lam,
fp_precision=fp_precision,
)
# Implementation of algo 3, quantile normalization
Hs_new = numbaList()
csums = numbaList()
ids_by_clusts = numbaList()
nbatch = len(Hs)
rg = np.random.default_rng(random_state)
seeds = rg.integers(4294967295, size=nbatch)
ref_batch = max_size = -1
for i in range(nbatch):
H_new = np.ascontiguousarray(Hs[i] / np.linalg.norm(Hs[i], axis=0),
dtype=np.float32) # Scale H
Hs_new.append(H_new) # Append scaled H
if not quantile_norm:
continue
clusters = np.argmax(H_new, axis=1) # Assign cluster
indices, _ = calculate_nearest_neighbors(
H_new, K=20, n_jobs=n_jobs, random_state=seeds[i]) # KNN with K=20
clusters, csum = _refine_cluster(clusters, indices,
n_components) # Refine cluster
csums.append(csum)
ids_by_clusts.append(np.argsort(clusters, kind='stable'))
if H_new.shape[0] > max_size: # Find ref batch
max_size = H_new.shape[0]
ref_batch = i
if quantile_norm:
_quantile_norm(Hs_new, csums, ids_by_clusts, nbatch, ref_batch,
n_components) # quantile normalization
data.uns["inmf_features"] = features # record which feature to use
data.uns["W"] = np.ascontiguousarray(
W.T, dtype=np.float32
) # cannot be varm because numbers of features are not the same
data.uns["V"] = np.array(Vs)
data.uns["inmf_err"] = err
data.obsm["H"] = np.concatenate(Hs)
data.obsm["X_inmf"] = np.concatenate(Hs_new)
return "inmf"
def _run_scrublet(data: Union[MultimodalData, UnimodalData],
name: Optional[str] = '',
expected_doublet_rate: Optional[float] = None,
sim_doublet_ratio: Optional[float] = 2.0,
n_prin_comps: Optional[int] = 30,
robust: Optional[bool] = False,
k: Optional[int] = None,
n_jobs: Optional[int] = -1,
random_state: Optional[int] = 0,
plot_hist: Optional[bool] = True) -> Union[None, "Figure"]:
"""Calculate doublet scores using Scrublet-like [Wolock18]_ strategy for the current data.X; determine a right threshold using Gaussian Mixture model.
This function should be called after highly_variable_gene selection.
Parameters
-----------
data: ``Union[MultimodalData, UnimodalData]`` object.
Annotated data matrix with rows for cells and columns for genes. Data must be low quality cell and gene filtered and log-transformed. Assume 'raw.X' stores the raw count matrix.
name: ``str``, optional, default: ``''``
Name of the sample.
expected_doublet_rate: ``float``, optional, default: ``None``
The expected doublet rate for the experiment. By default, calculate the expected rate based on number of cells from the 10x multiplet rate table
sim_doublet_ratio: ``float``, optional, default: ``2.0``
The ratio between synthetic doublets and observed cells.
n_prin_comps: ``int``, optional, default: ``30``
Number of principal components.
robust: ``bool``, optional, default: ``False``.
If true, use 'arpack' instead of 'randomized' for large matrices (i.e. max(X.shape) > 500 and n_components < 0.8 * min(X.shape))
k: ``int``, optional, default: ``None``
Number of observed cell neighbors. If None, k = round(0.5 * sqrt(number of observed cells)). Total neighbors k_adj = round(k * (1.0 + sim_doublet_ratio)).
n_job: ``int``, optional, default: ``-``
Number of threads to use. If ``-1``, use all available threads.
random_state: ``int``, optional, default: ``0``
Random state for doublet simulation, PCA and approximate nearest neighbor search.
plot_hist: ``bool``, optional, default: ``True``
If True, plot diagnostic histogram.
Returns
--------
``None`` or a ``matplotlib Figure object`` if
Update ``data.obs``:
* ``data.obs['doublet_score']``: The calculated doublet scores on cells.
* ``data.obs['pred_dbl']``: Predicted doublets as True.
Update ``data.uns``:
* ``data.uns['doublet_threshold']``: Inferred doublet threshold; any score > threshold is identified as a neotypic doublet.
Examples
--------
>>> pg.run_scrublet(data)
"""
from pegasus.tools import calculate_nearest_neighbors
from pegasus.cylib.fast_utils import simulate_doublets
from sklearn.decomposition import PCA
from scipy.stats import gaussian_kde
if "highly_variable_features" not in data.var:
raise ValueError(
"_run_scrublet must be run after highly_variable_features is called!"
)
r = sim_doublet_ratio
if expected_doublet_rate is None:
expected_doublet_rate = _calc_expected_doublet_rate(data.shape[0])
rho = expected_doublet_rate
# subset the raw count matrix
rawX = data.get_matrix("raw.X")
obs_umis = rawX.sum(axis=1, dtype=np.int32).A1
rawX = rawX[:, data.var["highly_variable_features"].values]
# Simulate synthetic doublets
sim_rawX, pair_idx = simulate_doublets(rawX, r, random_state)
sim_umis = obs_umis[pair_idx].sum(axis=1, dtype=np.int32)
# standardize and calculate PCA for rawX
obsX = rawX.astype(np.float32).toarray()
obsX /= obs_umis.reshape(-1, 1) # normalize each cell
m1 = obsX.mean(axis=0) # calculate mean and std
psum = np.multiply(obsX, obsX).sum(axis=0)
std = ((psum - obsX.shape[0] * (m1**2)) / (obsX.shape[0] - 1.0))**0.5
std[std == 0] = 1
obsX -= m1 # standardize
obsX /= std
svd_solver = "auto" if not robust else (
"arpack" if max(obsX.shape) > 500
and n_prin_comps < 0.8 * min(obsX.shape) else "full") # PCA
pca = PCA(n_components=n_prin_comps,
random_state=random_state,
svd_solver=svd_solver)
obs_pca = pca.fit_transform(obsX)
# standardize and calculate PCA for sim_rawX
simX = sim_rawX.astype(np.float32).toarray()
simX /= sim_umis.reshape(-1, 1) # normalize each cell
simX -= m1 # standardize
simX /= std
sim_pca = pca.transform(simX) # transform to PC coordinates
# concatenate observed and simulated data
pc_coords = np.vstack((obs_pca, sim_pca))
is_doublet = np.repeat(np.array([0, 1], dtype=np.int32),
[obsX.shape[0], simX.shape[0]])
# Calculate k nearest neighbors
if k is None:
k = int(round(0.5 * np.sqrt(obsX.shape[0])))
k_adj = int(round(k * (1.0 + r)))
indices, _ = calculate_nearest_neighbors(pc_coords,
K=k_adj + 1,
n_jobs=n_jobs)
# Calculate scrublet-like doublet score
k_d = is_doublet[indices].sum(axis=1)
q = (k_d + 1.0) / (k_adj + 2.0) # Equation 5
doublet_scores = (q * rho / r) / (
(1.0 - rho) - q * (1.0 - rho - rho / r)) # Equation 4
obs_scores = doublet_scores[0:obsX.shape[0]]
sim_scores = doublet_scores[obsX.shape[0]:]
# Determine a scrublet score threshold
# log transformed
sim_scores_log = np.log(sim_scores)
# Estimate KDE
min_score = sim_scores_log.min()
max_score = sim_scores_log.max()
min_gap = np.diff(np.unique(np.sort(sim_scores_log))).min()
from math import ceil
n_gap = max(int(ceil((max_score - min_score) / min_gap)),
200) # minimum is 200
gap = (max_score - min_score) / n_gap
n_ext = 5
min_score -= gap * n_ext
max_score += gap * n_ext
x = np.linspace(min_score, max_score,
n_gap + 1 + n_ext * 2) # generate x coordinates
kde = gaussian_kde(sim_scores_log)
y = kde(x)
# Find local maxima
maxima, maxima_by_x, filtered_maxima = _find_local_maxima(y)
assert maxima.size > 0
curv = _calc_vec_f(_curvature, x.size, y, gap) # calculate curvature
if maxima.size >= 2:
if maxima[0] < maxima[1]:
start = maxima[0]
end = maxima[1]
else:
start = maxima[1]
end = maxima[0]
pos = y[start + 1:end].argmin() + (start + 1)
else:
frac_right_thre = 0.42
frac_left_thre = 0.4
pos = -1
for i in range(maxima_by_x.size):
frac_right = (sim_scores_log >
x[maxima_by_x[i]]).sum() / sim_scores.size
if frac_right < frac_right_thre: # peak might represent a doublet peak, try to find a cutoff at the left side
if i == 0:
peak_curv_value = _find_curv_minima_at_peak(
curv, maxima_by_x[i])
end = _find_pos_curv(curv, maxima_by_x[i] - 1, '-')
start = _find_pos_curv(
curv,
_find_curv_local_minima(curv, peak_curv_value,
filtered_maxima, end - 1, '-')
+ 1, '+')
assert start <= end
pos = curv[start:end + 1].argmax() + start
else:
pos = y[maxima_by_x[i - 1] + 1:maxima_by_x[i]].argmin() + (
maxima_by_x[i - 1] + 1)
frac_left = (sim_scores_log < x[pos]).sum() / sim_scores.size
if frac_left < frac_left_thre:
pos = maxima_by_x[i]
break
if pos < 0:
# peak represents singlet, find a cutoff at the right side
peak_curv_value = _find_curv_minima_at_peak(curv, maxima_by_x[-1])
start = _find_pos_curv(curv, maxima_by_x[-1] + 1, '+')
end = _find_pos_curv(
curv,
_find_curv_local_minima(curv, peak_curv_value, filtered_maxima,
start + 1, '+') - 1, '-')
assert start <= end
pos = curv[start:end + 1].argmax() + start
threshold = np.exp(x[pos])
data.obs["doublet_score"] = obs_scores.astype(np.float32)
data.obs["pred_dbl"] = obs_scores > threshold
data.uns["doublet_threshold"] = float(threshold)
logger.info(
f"Sample {name}: doublet threshold = {threshold:.4f}; total cells = {data.shape[0]}; neotypic doublet rate = {data.obs['pred_dbl'].sum() / data.shape[0]:.2%}"
)
fig = None
if plot_hist:
fig = _plot_hist(obs_scores, sim_scores, threshold, x, y, curv)
return fig
def _run_scrublet(
data: Union[MultimodalData, UnimodalData],
raw_mat_key: Optional[str] = 'counts',
name: Optional[str] = '',
expected_doublet_rate: Optional[float] = None,
sim_doublet_ratio: Optional[float] = 2.0,
n_prin_comps: Optional[int] = 30,
k: Optional[int] = None,
n_jobs: Optional[int] = -1,
random_state: Optional[int] = 0,
plot_hist: Optional[bool] = True,
manual_correction: Optional[str] = None,
) -> Union[None, Figure]:
"""Calculate doublet scores using Scrublet-like [Wolock18]_ strategy for the current data.X; determine a right threshold based on the KDE curve.
This function should be called after highly_variable_gene selection.
Parameters
-----------
data: ``Union[MultimodalData, UnimodalData]`` object.
Annotated data matrix with rows for cells and columns for genes. Data must be low quality cell and gene filtered and log-transformed.
raw_mat_key: ``str``, optional, default: ``counts``
Matrix key for the raw count matrix.
name: ``str``, optional, default: ``''``
Name of the sample.
expected_doublet_rate: ``float``, optional, default: ``None``
The expected doublet rate for the experiment. By default, calculate the expected rate based on number of cells from the 10x multiplet rate table
sim_doublet_ratio: ``float``, optional, default: ``2.0``
The ratio between synthetic doublets and observed cells.
n_prin_comps: ``int``, optional, default: ``30``
Number of principal components.
k: ``int``, optional, default: ``None``
Number of observed cell neighbors. If None, k = round(0.5 * sqrt(number of observed cells)). Total neighbors k_adj = round(k * (1.0 + sim_doublet_ratio)).
n_jobs: ``int``, optional, default: ``-``
Number of threads to use. If ``-1``, use all physical CPU cores.
random_state: ``int``, optional, default: ``0``
Random state for doublet simulation, PCA and approximate nearest neighbor search.
plot_hist: ``bool``, optional, default: ``True``
If True, plot diagnostic histograms. Each sample would have a figure consisting of 4 panels showing histograms of doublet scores for observed cells (panel 1, density in log scale), simulated doublets (panel 2, density in log scale), KDE plot (panel 3) and signed curvature plot (panel 4) of log doublet scores for simulated doublets.
manual_correction: ``str``, optional, default: ``None``
If present, use human guide provided in manual_correction to select threshold. Currently only support manual_correction='peak', which means cut at the center of the peak.
Returns
--------
``None`` or a ``matplotlib Figure object`` if
Update ``data.obs``:
* ``data.obs['doublet_score']``: The calculated doublet scores on cells.
* ``data.obs['pred_dbl']``: Predicted doublets as True.
Update ``data.uns``:
* ``data.uns['doublet_threshold']``: Inferred doublet threshold; any score > threshold is identified as a neotypic doublet.
Examples
--------
>>> pg.run_scrublet(data)
"""
from pegasus.tools import calculate_nearest_neighbors, simulate_doublets
from sklearn.decomposition import PCA
from scipy.stats import gaussian_kde
from sklearn.cluster import KMeans
if "highly_variable_features" not in data.var:
raise ValueError(
"_run_scrublet must be run after highly_variable_features is called!"
)
r = sim_doublet_ratio
if expected_doublet_rate is None:
expected_doublet_rate = _calc_expected_doublet_rate(data.shape[0])
rho = expected_doublet_rate
# subset the raw count matrix
rawX = data.get_matrix(raw_mat_key)
obs_umis = rawX.sum(axis=1, dtype=np.int32).A1
rawX = rawX[:, data.var["highly_variable_features"].values]
# Simulate synthetic doublets
sim_rawX, pair_idx = simulate_doublets(rawX, r, random_state)
sim_umis = obs_umis[pair_idx].sum(axis=1, dtype=np.int32)
# standardize and calculate PCA for rawX
obsX = rawX.astype(np.float32).toarray()
obsX /= obs_umis.reshape(-1, 1) # normalize each cell
m1 = obsX.mean(axis=0) # calculate mean and std
psum = np.multiply(obsX, obsX).sum(axis=0)
std = ((psum - obsX.shape[0] * (m1**2)) / (obsX.shape[0] - 1.0))**0.5
std[std == 0] = 1
obsX -= m1 # standardize
obsX /= std
pca = PCA(n_components=n_prin_comps, random_state=random_state)
n_jobs = eff_n_jobs(n_jobs)
with threadpool_limits(limits=n_jobs):
obs_pca = pca.fit_transform(obsX.astype(
np.float64)) # float64 for reproducibility
obs_pca = np.ascontiguousarray(obs_pca, dtype=np.float32)
kmeans = KMeans(n_clusters=5, random_state=random_state).fit(obs_pca)
# calculate in simulated distribution, expected percentage of embedded doublets
data.obs["dbl_kmeans_"] = pd.Categorical(kmeans.labels_)
_, freqs = np.unique(kmeans.labels_, return_counts=True)
freqs = np.array(freqs) / sum(freqs)
d_emb = (((1.0 - rho) * freqs + rho * (freqs**2))**2).sum()
d_neo = 1.0 - d_emb
# standardize and calculate PCA for sim_rawX
simX = sim_rawX.astype(np.float32).toarray()
simX /= sim_umis.reshape(-1, 1) # normalize each cell
simX -= m1 # standardize
simX /= std
sim_pca = pca.transform(simX) # transform to PC coordinates
sim_pca = np.ascontiguousarray(sim_pca, dtype=np.float32)
# concatenate observed and simulated data
pc_coords = np.vstack((obs_pca, sim_pca))
is_doublet = np.repeat(np.array([0, 1], dtype=np.int32),
[obsX.shape[0], simX.shape[0]])
# Calculate k nearest neighbors
if k is None:
k = int(round(0.5 * np.sqrt(obsX.shape[0])))
k_adj = int(round(k * (1.0 + r)))
indices, _ = calculate_nearest_neighbors(pc_coords,
K=k_adj + 1,
n_jobs=n_jobs)
# Calculate scrublet-like doublet score
k_d = is_doublet[indices].sum(axis=1)
q = (k_d + 1.0) / (k_adj + 2.0) # Equation 5
doublet_scores = (q * rho / r) / (
(1.0 - rho) - q * (1.0 - rho - rho / r)) # Equation 4
obs_scores = doublet_scores[0:obsX.shape[0]]
sim_scores = doublet_scores[obsX.shape[0]:]
# Determine a scrublet score threshold
# log transformed
sim_scores_log = np.log(sim_scores)
# Estimate KDE
min_score = sim_scores_log.min()
max_score = sim_scores_log.max()
min_gap = np.diff(np.unique(np.sort(sim_scores_log))).min()
from math import ceil
n_gap = max(int(ceil((max_score - min_score) / min_gap)),
200) # minimum is 200
gap = (max_score - min_score) / n_gap
n_ext = 5
min_score -= gap * n_ext
max_score += gap * n_ext
x = np.linspace(min_score, max_score,
n_gap + 1 + n_ext * 2) # generate x coordinates
kde = gaussian_kde(sim_scores_log)
y = kde(x)
# Find local maxima
maxima, maxima_by_x, filtered_maxima = _find_local_maxima(y)
assert maxima.size > 0
curv = _calc_vec_f(_curvature, x.size, y, gap) # calculate curvature
x_theory = np.percentile(sim_scores_log, d_emb * 100.0 + 1e-6)
threshold_theory = np.exp(x_theory)
case_num = -1
pos = -1
if maxima.size >= 2:
pos = _locate_cutoff_among_peaks_with_guide(x, y, maxima,
sim_scores_log, d_neo)
case_num = 0
d_pneo = (sim_scores_log > x[pos]).sum() / sim_scores_log.size
if d_pneo < 0.1: # < 10%, consider it as not a peak
idx_ = maxima_by_x >= pos
filtered_maxima = np.concatenate(
(filtered_maxima, maxima_by_x[idx_]))
maxima_by_x = maxima_by_x[~idx_]
pos = -1
if pos < 0:
frac_right = (sim_scores_log >
x[maxima_by_x[-1]]).sum() / sim_scores.size
if frac_right < 0.41 or (frac_right < 0.5
and x_theory + 0.05 < x[maxima_by_x[-1]]):
logger.debug(f"frac_right={frac_right}.")
if maxima_by_x.size > 1:
posvec = np.vectorize(
lambda i: y[maxima_by_x[i] + 1:maxima_by_x[i + 1]].argmin(
) + (maxima_by_x[i] + 1))(range(maxima_by_x.size - 1))
pos = posvec[np.argmin(np.abs(x[posvec] - x_theory))]
case_num = 1
else:
pos = _find_cutoff_left_side(maxima_by_x[0], x, curv, x_theory)
case_num = 2
else:
pos = _find_cutoff_right_side(maxima_by_x[-1], curv,
filtered_maxima)
case_num = 3
threshold = np.exp(x[pos])
threshold_auto = None
if manual_correction is not None:
assert case_num == 2
threshold_auto = threshold
threshold = np.exp(x[maxima_by_x[-1]])
data.obs["doublet_score"] = obs_scores.astype(np.float32)
data.obs["pred_dbl"] = obs_scores > threshold
data.uns["doublet_threshold"] = float(threshold)
neo_dbl_rate = data.obs['pred_dbl'].sum() / data.shape[0]
neo_sim_dbl_rate = (sim_scores > threshold).sum() / sim_scores.size
logger.info(
f"Sample {name}: doublet threshold = {threshold:.4f}; total cells = {data.shape[0]}; neotypic doublet rate in simulation = {neo_sim_dbl_rate:.2%}; neotypic doublet rate = {neo_dbl_rate:.2%}."
)
fig = None
if plot_hist:
fig = _plot_hist(obs_scores,
sim_scores,
threshold,
threshold_theory,
x,
y,
curv,
threshold_auto=threshold_auto)
return fig
def net_umap(
data: MultimodalData,
rep: str = "pca",
n_jobs: int = -1,
n_components: int = 2,
n_neighbors: int = 15,
min_dist: float = 0.5,
spread: float = 1.0,
densmap: bool = False,
dens_lambda: float = 2.0,
dens_frac: float = 0.3,
dens_var_shift: float = 0.1,
random_state: int = 0,
select_frac: float = 0.1,
select_K: int = 25,
select_alpha: float = 1.0,
full_speed: bool = False,
net_alpha: float = 0.1,
polish_learning_rate: float = 10.0,
polish_n_epochs: int = 30,
out_basis: str = "net_umap",
) -> None:
"""Calculate Net-UMAP embedding of cells.
Net-UMAP is an approximated UMAP embedding using Deep Learning model to improve the speed.
In specific, the deep model used is MLPRegressor_, the *scikit-learn* implementation of Multi-layer Perceptron regressor.
See [Li20]_ for details.
.. _MLPRegressor: https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPRegressor.html
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
rep: ``str``, optional, default: ``"pca"``
Representation of data used for the calculation. By default, use PCA coordinates. If ``None``, use the count matrix ``data.X``.
n_jobs: ``int``, optional, default: ``-1``
Number of threads to use. If ``-1``, use all physical CPU cores.
n_components: ``int``, optional, default: ``2``
Dimension of calculated UMAP coordinates. By default, generate 2-dimensional data for 2D visualization.
n_neighbors: ``int``, optional, default: ``15``
Number of nearest neighbors considered during the computation.
min_dist: ``float``, optional, default: ``0.5``
The effective minimum distance between embedded data points.
spread: ``float``, optional, default: ``1.0``
The effective scale of embedded data points.
densmap: ``bool``, optional, default: ``False``
Whether the density-augmented objective of densMAP should be used for optimization, which will generate an embedding where
local densities are encouraged to be correlated with those in the original space.
dens_lambda: ``float``, optional, default: ``2.0``
Controls the regularization weight of the density correlation term in densMAP. Only works when *densmap* is ``True``.
Larger values prioritize density preservation over the UMAP objective, while values closer to 0 for the opposite direction.
Notice that setting this parameter to ``0`` is equivalent to running the original UMAP algorithm.
dens_frac: ``float``, optional, default: ``0.3``
Controls the fraction of epochs (between 0 and 1) where the density-augmented objective is used in densMAP. Only works when
*densmap* is ``True``.
The first ``(1 - dens_frac)`` fraction of epochs optimize the original UMAP objective before introducing the density
correlation term.
dens_var_shift: ``float``, optional, default, ``0.1``
A small constant added to the variance of local radii in the embedding when calculating the density correlation objective to
prevent numerical instability from dividing by a small number. Only works when *densmap* is ``True``.
random_state: ``int``, optional, default: ``0``
Random seed set for reproducing results.
select_frac: ``float``, optional, default: ``0.1``
Down sampling fraction on the cells.
select_K: ``int``, optional, default: ``25``
Number of neighbors to be used to estimate local density for each data point for down sampling.
select_alpha: ``float``, optional, default: ``1.0``
Weight the down sample to be proportional to ``radius ** select_alpha``.
full_speed: ``bool``, optional, default: ``False``
* If ``True``, use multiple threads in constructing ``hnsw`` index. However, the kNN results are not reproducible.
* Otherwise, use only one thread to make sure results are reproducible.
net_alpha: ``float``, optional, default: ``0.1``
L2 penalty (regularization term) parameter of the deep regressor.
polish_learning_frac: ``float``, optional, default: ``10.0``
After running the deep regressor to predict new coordinates, use ``polish_learning_frac`` * ``n_obs`` as the learning rate to polish the coordinates.
polish_n_iter: ``int``, optional, default: ``30``
Number of iterations for polishing UMAP run.
out_basis: ``str``, optional, default: ``"net_umap"``
Key name for calculated UMAP coordinates to store.
Returns
-------
``None``
Update ``data.obsm``:
* ``data.obsm['X_' + out_basis]``: Net UMAP coordinates of the data.
Update ``data.obs``:
* ``data.obs['ds_selected']``: Boolean array to indicate which cells are selected during the down sampling phase.
Examples
--------
>>> pg.net_umap(data)
"""
rep = update_rep(rep)
n_jobs = eff_n_jobs(n_jobs)
knn_indices, knn_dists = get_neighbors(data,
K=select_K,
rep=rep,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed)
selected = select_cells(
knn_dists,
select_frac,
K=select_K,
alpha=select_alpha,
random_state=random_state,
)
X_full = X_from_rep(data, rep)
X = X_full[selected, :]
if data.shape[0] < n_neighbors:
logger.warning(
f"Warning: Number of samples = {data.shape[0]} < K = {n_neighbors}!\n Set K to {data.shape[0]}."
)
n_neighbors = data.shape[0]
ds_indices_key = "ds_" + rep + "_knn_indices" # ds refers to down-sampling
ds_distances_key = "ds_" + rep + "_knn_distances"
indices, distances = calculate_nearest_neighbors(
X,
K=n_neighbors,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed,
)
data.uns[ds_indices_key] = indices
data.uns[ds_distances_key] = distances
knn_indices = np.insert(data.uns[ds_indices_key][:, 0:n_neighbors - 1],
0,
range(X.shape[0]),
axis=1)
knn_dists = np.insert(data.uns[ds_distances_key][:, 0:n_neighbors - 1],
0,
0.0,
axis=1)
X_umap = calc_umap(
X,
n_components=n_components,
n_neighbors=n_neighbors,
min_dist=min_dist,
spread=spread,
densmap=densmap,
dens_lambda=dens_lambda,
dens_frac=dens_frac,
dens_var_shift=dens_var_shift,
random_state=random_state,
knn_indices=knn_indices,
knn_dists=knn_dists,
)
data.uns["X_" + out_basis + "_small"] = X_umap
data.obs["ds_selected"] = selected
Y_init = np.zeros((data.shape[0], n_components), dtype=np.float64)
Y_init[selected, :] = X_umap
Y_init[~selected, :] = net_train_and_predict(X,
X_umap,
X_full[~selected, :],
net_alpha,
n_jobs,
random_state,
verbose=True)
data.obsm["X_" + out_basis + "_pred"] = Y_init
knn_indices, knn_dists = get_neighbors(data,
K=n_neighbors,
rep=rep,
n_jobs=n_jobs,
random_state=random_state,
full_speed=full_speed)
knn_indices = np.insert(knn_indices[:, 0:n_neighbors - 1],
0,
range(data.shape[0]),
axis=1)
knn_dists = np.insert(knn_dists[:, 0:n_neighbors - 1], 0, 0.0, axis=1)
key = f"X_{out_basis}"
data.obsm[key] = calc_umap(
X_full,
n_components=n_components,
n_neighbors=n_neighbors,
min_dist=min_dist,
spread=spread,
densmap=densmap,
dens_lambda=dens_lambda,
dens_frac=dens_frac,
dens_var_shift=dens_var_shift,
random_state=random_state,
init=Y_init,
n_epochs=polish_n_epochs,
learning_rate=polish_learning_rate,
knn_indices=knn_indices,
knn_dists=knn_dists,
)
data.register_attr(key, "basis")