def write_loom_file(data: MultimodalData, output_file: str) -> None:
""" Write a MultimodalData to loom file. Will assert data only contain one type of experiment.
"""
keys = data.list_data()
if len(keys) > 1:
raise ValueError(
f"Data contain multiple modalities: {','.join(keys)}!")
data.select_data(keys[0])
matrices = data.list_keys()
assert "X" in matrices
if len(matrices) == 0:
raise ValueError("Could not write empty matrix to a loom file!")
def _process_attrs(key_name: str, attrs: pd.DataFrame,
attrs_multi: dict) -> Dict[str, object]:
res_dict = {key_name: attrs.index.values}
for key in attrs.columns:
res_dict[key] = np.array(attrs[key].values)
for key, value in attrs_multi.items():
if value.ndim > 1: # value.ndim == 1 refers to np.recarray, which will not be written to a loom file.
res_dict[key] = value if value.shape[1] > 1 else value[:, 0]
return res_dict
row_attrs = _process_attrs("Gene", data.var, data.varm)
col_attrs = _process_attrs("CellID", data.obs, data.obsm)
accession_key = "featureid" if "featureid" in row_attrs else (
"gene_ids" if "gene_ids" in row_attrs else None)
if accession_key is not None:
row_attrs["Accession"] = row_attrs.pop(accession_key)
layers = {}
for matkey in matrices:
layers["" if matkey == "X" else matkey] = data.get_matrix(matkey).T
file_attrs = {}
for key, value in data.uns.items():
if isinstance(value, str):
file_attrs[key] = value
import loompy
loompy.create(output_file,
layers,
row_attrs,
col_attrs,
file_attrs=file_attrs)
logger.info(f"{output_file} is written.")
def log_norm(
data: MultimodalData,
norm_count: float = 1e5,
backup_matrix: str = "raw.X",
) -> None:
"""Normalization, and then apply natural logarithm to the data.
Parameters
----------
data: ``pegasusio.MultimodalData``
Use current selected modality in data, which should contain one RNA expression matrix.
norm_count: ``int``, optional, default: ``1e5``.
Total counts of one cell after normalization.
backup_matrix: ``str``, optional, default: ``raw.X``.
The key name of the backup count matrix, usually the raw counts.
Returns
-------
``None``
Update ``data.X`` with count matrix after log-normalization. In addition, back up the original count matrix as ``backup_matrix``.
In case of rerunning normalization while ``backup_matrix`` already exists, use ``backup_matrix`` instead of ``data.X`` for normalization.
Examples
--------
>>> pg.log_norm(data)
"""
if isinstance(data, MultimodalData):
data = data.current_data()
assert data.get_modality() == "rna"
if backup_matrix not in data.list_keys():
data.add_matrix(backup_matrix, data.X)
data.X = data.X.astype(np.float32) # force copy
else:
# The case of rerunning log_norm. Use backup matrix as source.
data.X = data.get_matrix(backup_matrix).astype(
np.float32) # force copy
logger.warning(
"Rerun log-normalization. Use the raw counts in backup instead.")
data.obs["scale"] = normalize_by_count(data.X, data.var["robust"].values,
norm_count, True)
data.uns["norm_count"] = norm_count
def infer_doublets(
data: MultimodalData,
channel_attr: Optional[str] = None,
clust_attr: Optional[str] = None,
min_cell: Optional[int] = 100,
expected_doublet_rate: Optional[float] = None,
sim_doublet_ratio: Optional[float] = 2.0,
n_prin_comps: Optional[int] = 30,
robust: Optional[bool] = False,
k: Optional[int] = None,
n_jobs: Optional[int] = -1,
alpha: Optional[float] = 0.05,
random_state: Optional[int] = 0,
plot_hist: Optional[str] = "dbl",
) -> None:
"""Infer doublets using a Scrublet-like strategy. [Li20-2]_
This function must be called after clustering.
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
channel_attr: ``str``, optional, default: None
Attribute indicating sample channels. If set, calculate scrublet-like doublet scores per channel.
clust_attr: ``str``, optional, default: None
Attribute indicating cluster labels. If set, estimate proportion of doublets in each cluster and statistical significance.
min_cell: ``int``, optional, default: 100
Minimum number of cells per sample to calculate doublet scores. For samples having less than 'min_cell' cells, doublet score calculation will be skipped.
expected_doublet_rate: ``float``, optional, default: ``None``
The expected doublet rate for the experiment. By default, calculate the expected rate based on number of cells from the 10x multiplet rate table
sim_doublet_ratio: ``float``, optional, default: ``2.0``
The ratio between synthetic doublets and observed cells.
n_prin_comps: ``int``, optional, default: ``30``
Number of principal components.
robust: ``bool``, optional, default: ``False``.
If true, use 'arpack' instead of 'randomized' for large matrices (i.e. max(X.shape) > 500 and n_components < 0.8 * min(X.shape))
k: ``int``, optional, default: ``None``
Number of observed cell neighbors. If None, k = round(0.5 * sqrt(number of observed cells)). Total neighbors k_adj = round(k * (1.0 + sim_doublet_ratio)).
n_job: ``int``, optional, default: ``-``
Number of threads to use. If ``-1``, use all available threads.
alpha: ``float``, optional, default: ``0.05``
FDR significant level for cluster-level fisher exact test.
random_state: ``int``, optional, default: ``0``
Random seed for reproducing results.
plot_hist: ``str``, optional, default: ``dbl``
If not None, plot diagnostic histograms using ``plot_hist`` as the prefix. If `channel_attr` is None, ``plot_hist.png`` is generated; Otherwise, ``plot_hist.channel_name.png`` files are generated.
Returns
-------
``None``
Update ``data.obs``:
* ``data.obs['pred_dbl_type']``: Predicted singlet/doublet types.
* ``data.uns['pred_dbl_cluster']``: Only generated if 'clust_attr' is not None. This is a dataframe with two columns, 'Cluster' and 'Qval'. Only clusters with significantly more doublets than expected will be recorded here.
Examples
--------
>>> pg.infer_doublets(data, channel_attr = 'Channel', clust_attr = 'Annotation')
"""
assert data.get_modality() == "rna"
try:
rawX = data.get_matrix("raw.X")
except ValueError:
raise ValueError(
"Cannot detect the raw count matrix raw.X; stop inferring doublets!"
)
if_plot = plot_hist is not None
if channel_attr is None:
if data.shape[0] >= min_cell:
fig = _run_scrublet(data, expected_doublet_rate = expected_doublet_rate, sim_doublet_ratio = sim_doublet_ratio, \
n_prin_comps = n_prin_comps, robust = robust, k = k, n_jobs = n_jobs, random_state = random_state, \
plot_hist = if_plot)
if if_plot:
fig.savefig(f"{plot_hist}.png")
else:
logger.warning(
f"Data has {data.shape[0]} < {min_cell} cells and thus doublet score calculation is skipped!"
)
data.obs["doublet_score"] = 0.0
data.obs["pred_dbl"] = False
else:
from pandas.api.types import is_categorical_dtype
from pegasus.tools import identify_robust_genes, log_norm, highly_variable_features
assert is_categorical_dtype(data.obs[channel_attr])
genome = data.get_genome()
modality = data.get_modality()
channels = data.obs[channel_attr].cat.categories
dbl_score = np.zeros(data.shape[0], dtype=np.float32)
pred_dbl = np.zeros(data.shape[0], dtype=np.bool_)
thresholds = {}
for channel in channels:
# Generate a new unidata object for the channel
idx = np.where(data.obs[channel_attr] == channel)[0]
if idx.size >= min_cell:
unidata = UnimodalData({"barcodekey": data.obs_names[idx]},
{"featurekey": data.var_names},
{"X": rawX[idx]}, {
"genome": genome,
"modality": modality
})
# Identify robust genes, count and log normalized and select top 2,000 highly variable features
identify_robust_genes(unidata)
log_norm(unidata)
highly_variable_features(unidata)
# Run _run_scrublet
fig = _run_scrublet(unidata, name = channel, expected_doublet_rate = expected_doublet_rate, sim_doublet_ratio = sim_doublet_ratio, \
n_prin_comps = n_prin_comps, robust = robust, k = k, n_jobs = n_jobs, random_state = random_state, \
plot_hist = if_plot)
if if_plot:
fig.savefig(f"{plot_hist}.{channel}.png")
dbl_score[idx] = unidata.obs["doublet_score"].values
pred_dbl[idx] = unidata.obs["pred_dbl"].values
thresholds[channel] = unidata.uns["doublet_threshold"]
else:
logger.warning(
f"Channel {channel} has {idx.size} < {min_cell} cells and thus doublet score calculation is skipped!"
)
data.obs["doublet_score"] = dbl_score
data.obs["pred_dbl"] = pred_dbl
data.uns["doublet_thresholds"] = thresholds
if clust_attr is not None:
data.uns["pred_dbl_cluster"] = _identify_doublets_fisher(
data.obs[clust_attr].values,
data.obs["pred_dbl"].values,
alpha=alpha)
logger.info('Doublets are predicted!')
def infer_doublets(
data: MultimodalData,
channel_attr: Optional[str] = None,
clust_attr: Optional[str] = None,
raw_mat_key: Optional[str] = 'counts',
min_cell: Optional[int] = 100,
expected_doublet_rate: Optional[float] = None,
sim_doublet_ratio: Optional[float] = 2.0,
n_prin_comps: Optional[int] = 30,
k: Optional[int] = None,
n_jobs: Optional[int] = -1,
alpha: Optional[float] = 0.05,
random_state: Optional[int] = 0,
plot_hist: Optional[str] = "sample",
manual_correction: Optional[str] = None,
) -> None:
"""Infer doublets by first calculating Scrublet-like [Wolock18]_ doublet scores and then smartly determining an appropriate doublet score cutoff [Li20-2]_ .
This function should be called after clustering if clust_attr is not None. In this case, we will test if each cluster is significantly enriched for doublets using Fisher's exact test.
Parameters
----------
data: ``pegasusio.MultimodalData``
Annotated data matrix with rows for cells and columns for genes.
channel_attr: ``str``, optional, default: None
Attribute indicating sample channels. If set, calculate scrublet-like doublet scores per channel.
clust_attr: ``str``, optional, default: None
Attribute indicating cluster labels. If set, estimate proportion of doublets in each cluster and statistical significance.
min_cell: ``int``, optional, default: 100
Minimum number of cells per sample to calculate doublet scores. For samples having less than 'min_cell' cells, doublet score calculation will be skipped.
expected_doublet_rate: ``float``, optional, default: ``None``
The expected doublet rate for the experiment. By default, calculate the expected rate based on number of cells from the 10x multiplet rate table
sim_doublet_ratio: ``float``, optional, default: ``2.0``
The ratio between synthetic doublets and observed cells.
n_prin_comps: ``int``, optional, default: ``30``
Number of principal components.
k: ``int``, optional, default: ``None``
Number of observed cell neighbors. If None, k = round(0.5 * sqrt(number of observed cells)). Total neighbors k_adj = round(k * (1.0 + sim_doublet_ratio)).
n_jobs: ``int``, optional, default: ``-1``
Number of threads to use. If ``-1``, use all physical CPU cores.
alpha: ``float``, optional, default: ``0.05``
FDR significant level for cluster-level fisher exact test.
random_state: ``int``, optional, default: ``0``
Random seed for reproducing results.
plot_hist: ``str``, optional, default: ``sample``
If not None, plot diagnostic histograms using ``plot_hist`` as the prefix. If `channel_attr` is None, ``plot_hist.dbl.png`` is generated; Otherwise, ``plot_hist.channel_name.dbl.png`` files are generated. Each figure consists of 4 panels showing histograms of doublet scores for observed cells (panel 1, density in log scale), simulated doublets (panel 2, density in log scale), KDE plot (panel 3) and signed curvature plot (panel 4) of log doublet scores for simulated doublets. Each plot contains two dashed lines. The red dashed line represents the theoretical cutoff (calucalted based on number of cells and 10x doublet table) and the black dashed line represents the cutof inferred from the data.
manual_correction: ``str``, optional, default: ``None``
Use human guide to correct doublet threshold for certain channels. This is string representing a comma-separately list. Each item in the list represent one sample and the sample name and correction guide are separated using ':'. The only correction guide supported is 'peak', which means cut at the center of the peak. If only one sample available, use '' as the sample name.
Returns
-------
``None``
Update ``data.obs``:
* ``data.obs['pred_dbl']``: Predicted singlet/doublet types.
* ``data.uns['pred_dbl_cluster']``: Only generated if 'clust_attr' is not None. This is a dataframe with two columns, 'Cluster' and 'Qval'. Only clusters with significantly more doublets than expected will be recorded here.
Examples
--------
>>> pg.infer_doublets(data, channel_attr = 'Channel', clust_attr = 'Annotation')
"""
assert data.get_modality() == "rna"
try:
rawX = data.get_matrix(raw_mat_key)
except ValueError:
raise ValueError(
f"Cannot detect the raw count matrix {raw_mat_key}; stop inferring doublets!"
)
if_plot = plot_hist is not None
mancor = {}
if manual_correction is not None:
for item in manual_correction.split(','):
name, action = item.split(':')
mancor[name] = action
if channel_attr is None:
if data.shape[0] >= min_cell:
fig = _run_scrublet(data, raw_mat_key, expected_doublet_rate = expected_doublet_rate, sim_doublet_ratio = sim_doublet_ratio, \
n_prin_comps = n_prin_comps, k = k, n_jobs = n_jobs, random_state = random_state, plot_hist = if_plot, manual_correction = mancor.get('', None))
if if_plot:
fig.savefig(f"{plot_hist}.dbl.png")
else:
logger.warning(
f"Data has {data.shape[0]} < {min_cell} cells and thus doublet score calculation is skipped!"
)
data.obs["doublet_score"] = 0.0
data.obs["pred_dbl"] = False
else:
from pandas.api.types import is_categorical_dtype
from pegasus.tools import identify_robust_genes, log_norm, highly_variable_features
assert is_categorical_dtype(data.obs[channel_attr])
genome = data.get_genome()
modality = data.get_modality()
channels = data.obs[channel_attr].cat.categories
dbl_score = np.zeros(data.shape[0], dtype=np.float32)
pred_dbl = np.zeros(data.shape[0], dtype=np.bool_)
thresholds = {}
for channel in channels:
# Generate a new unidata object for the channel
idx = np.where(data.obs[channel_attr] == channel)[0]
if idx.size >= min_cell:
unidata = UnimodalData({"barcodekey": data.obs_names[idx]},
{"featurekey": data.var_names},
{"counts": rawX[idx]}, {
"genome": genome,
"modality": modality
},
cur_matrix="counts")
# Identify robust genes, count and log normalized and select top 2,000 highly variable features
identify_robust_genes(unidata)
log_norm(unidata)
highly_variable_features(unidata)
# Run _run_scrublet
fig = _run_scrublet(unidata, raw_mat_key, name = channel, expected_doublet_rate = expected_doublet_rate, sim_doublet_ratio = sim_doublet_ratio, \
n_prin_comps = n_prin_comps, k = k, n_jobs = n_jobs, random_state = random_state, plot_hist = if_plot, manual_correction = mancor.get(channel, None))
if if_plot:
fig.savefig(f"{plot_hist}.{channel}.dbl.png")
dbl_score[idx] = unidata.obs["doublet_score"].values
pred_dbl[idx] = unidata.obs["pred_dbl"].values
thresholds[channel] = unidata.uns["doublet_threshold"]
else:
logger.warning(
f"Channel {channel} has {idx.size} < {min_cell} cells and thus doublet score calculation is skipped!"
)
data.obs["doublet_score"] = dbl_score
data.obs["pred_dbl"] = pred_dbl
data.uns["doublet_thresholds"] = thresholds
if clust_attr is not None:
data.uns["pred_dbl_cluster"] = _identify_doublets_fisher(
data.obs[clust_attr].values,
data.obs["pred_dbl"].values,
alpha=alpha)
logger.info('Doublets are predicted!')
def pseudobulk(
data: MultimodalData,
sample: str,
attrs: Optional[Union[List[str], str]] = None,
mat_key: Optional[str] = "counts",
cluster: Optional[str] = None,
) -> UnimodalData:
"""Generate Pseudo-bulk count matrices.
Parameters
-----------
data: ``MultimodalData`` or ``UnimodalData`` object
Annotated data matrix with rows for cells and columns for genes.
sample: ``str``
Specify the cell attribute used for aggregating pseudo-bulk data.
Key must exist in ``data.obs``.
attrs: ``str`` or ``List[str]``, optional, default: ``None``
Specify additional cell attributes to remain in the pseudo bulk data.
If set, all attributes' keys must exist in ``data.obs``.
Notice that for a categorical attribute, each pseudo-bulk's value is the one of highest frequency among its cells,
and for a numeric attribute, each pseudo-bulk's value is the mean among its cells.
mat_key: ``str``, optional, default: ``counts``
Specify the single-cell count matrix used for aggregating pseudo-bulk counts:
If specified, use the count matrix with key ``mat_key`` from matrices of ``data``; otherwise, default is ``counts``.
cluster: ``str``, optional, default: ``None``
If set, additionally generate pseudo-bulk matrices per cluster specified in ``data.obs[cluster]``.
Returns
-------
A UnimodalData object ``udata`` containing pseudo-bulk information:
* It has the following count matrices:
* ``X``: The pseudo-bulk count matrix over all cells.
* If ``cluster`` is set, a number of pseudo-bulk count matrices of cells belonging to the clusters, respectively.
* ``udata.obs``: It contains pseudo-bulk attributes aggregated from the corresponding single-cell attributes.
* ``udata.var``: Gene names and Ensembl IDs are maintained.
Update ``data``:
* Add the returned UnimodalData object above to ``data`` with key ``<sample>-pseudobulk``, where ``<sample>`` is replaced by the actual value of ``sample`` argument.
Examples
--------
>>> pg.pseudobulk(data, sample="Channel")
"""
X = data.get_matrix(mat_key)
assert sample in data.obs.columns, f"Sample key '{sample}' must exist in data.obs!"
sample_vec = (data.obs[sample] if is_categorical_dtype(data.obs[sample])
else data.obs[sample].astype("category"))
bulk_list = sample_vec.cat.categories
df_barcode = data.obs.reset_index()
mat_dict = {
"counts": get_pseudobulk_count(X, df_barcode, sample, bulk_list)
}
# Generate pseudo-bulk attributes if specified
bulk_attr_list = []
if attrs is not None:
if isinstance(attrs, str):
attrs = [attrs]
for attr in attrs:
assert (attr in data.obs.columns
), f"Cell attribute key '{attr}' must exist in data.obs!"
for bulk in bulk_list:
df_bulk = df_barcode.loc[df_barcode[sample] == bulk]
if attrs is not None:
bulk_attr = df_bulk[attrs].apply(set_bulk_value, axis=0)
bulk_attr["barcodekey"] = bulk
else:
bulk_attr = pd.Series({"barcodekey": bulk})
bulk_attr_list.append(bulk_attr)
df_pseudobulk = pd.DataFrame(bulk_attr_list)
df_feature = pd.DataFrame(index=data.var_names)
if "featureid" in data.var.columns:
df_feature["featureid"] = data.var["featureid"]
if cluster is not None:
assert (cluster in data.obs.columns
), f"Cluster key '{attr}' must exist in data.obs!"
cluster_list = data.obs[cluster].astype("category").cat.categories
for cls in cluster_list:
mat_dict[f"{cluster}_{cls}.X"] = get_pseudobulk_count(
X, df_barcode.loc[df_barcode[cluster] == cls], sample,
bulk_list)
udata = UnimodalData(
barcode_metadata=df_pseudobulk,
feature_metadata=df_feature,
matrices=mat_dict,
genome=sample,
modality="pseudobulk",
cur_matrix="counts",
)
data.add_data(udata)
return udata
