def filter_and_normalize(adata, min_counts=10, n_top_genes=None, log=True, copy=False): """Filtering, normalization and log transform Expects non-logarithmized data. If using logarithmized data, pass `log=False`. Runs the following steps .. code:: python sc.pp.filter_genes(adata, min_counts=10) sc.pp.normalize_per_cell(adata) sc.pp.filter_genes_dispersion(adata, n_top_genes=10000) sc.pp.normalize_per_cell(adata) if log: sc.pp.log1p(adata) Arguments --------- adata: :class:`~anndata.AnnData` Annotated data matrix. min_counts: `int` (default: 10) Minimum number of gene counts per cell. n_top_genes: `int` (default: 10000) Number of genes to keep. log: `bool` (default: `True`) Take logarithm. copy: `bool` (default: `False`) Return a copy of `adata` instead of updating it. Returns ------- Returns or updates `adata` depending on `copy`. """ from scanpy.api.pp import filter_genes, filter_genes_dispersion, normalize_per_cell, log1p filter_genes(adata, min_counts=min_counts) if n_top_genes is not None and n_top_genes < adata.shape[1]: normalize_per_cell(adata) filter_genes_dispersion(adata, n_top_genes=n_top_genes) normalize_per_cell(adata) if log: log1p(adata) return adata if copy else None
def filter_and_normalize(data, min_counts=None, min_counts_u=None, min_cells=None, min_cells_u=None, min_shared_counts=None, min_shared_cells=None, n_top_genes=None, flavor='seurat', log=True, copy=False): """Filtering, normalization and log transform Expects non-logarithmized data. If using logarithmized data, pass `log=False`. Runs the following steps .. code:: python scv.pp.filter_genes(adata) scv.pp.normalize_per_cell(adata) if n_top_genes is not None: scv.pp.filter_genes_dispersion(adata) if log: scv.pp.log1p(adata) Arguments --------- data: :class:`~anndata.AnnData` Annotated data matrix. min_counts: `int` (default: `None`) Minimum number of counts required for a gene to pass filtering (spliced). min_counts_u: `int` (default: `None`) Minimum number of counts required for a gene to pass filtering (unspliced). min_cells: `int` (default: `None`) Minimum number of cells expressed required for a gene to pass filtering (spliced). min_cells_u: `int` (default: `None`) Minimum number of cells expressed required for a gene to pass filtering (unspliced). min_shared_counts: `int`, optional (default: `None`) Minimum number of counts (in cells expressed simultaneously in unspliced and spliced) required for a gene. min_shared_cells: `int`, optional (default: `None`) Minimum number of cells required for a gene to be expressed simultaneously in unspliced and spliced. n_top_genes: `int` (default: `None`) Number of genes to keep. flavor: {'seurat', 'cell_ranger', 'svr'}, optional (default: 'seurat') Choose the flavor for computing normalized dispersion. If choosing 'seurat', this expects non-logarithmized data. log: `bool` (default: `True`) Take logarithm. copy: `bool` (default: `False`) Return a copy of `adata` instead of updating it. Returns ------- Returns or updates `adata` depending on `copy`. """ adata = data.copy() if copy else data if 'spliced' not in adata.layers.keys( ) or 'unspliced' not in adata.layers.keys(): raise ValueError('Could not find spliced / unspliced counts.') filter_genes( adata, min_counts=min_counts, min_counts_u=min_counts_u, min_cells=min_cells, min_cells_u=min_cells_u, min_shared_counts=min_shared_counts, min_shared_cells=min_shared_cells, ) normalize_per_cell(adata) if n_top_genes is not None: filter_genes_dispersion(adata, n_top_genes=n_top_genes, flavor=flavor) log_advised = np.allclose(adata.X[:10].sum(), adata.layers['spliced'][:10].sum()) if log and log_advised: log1p(adata) if log and log_advised: logg.info('Logarithmized X.') elif log and not log_advised: logg.warn('Did not modify X as it looks preprocessed already.') elif log_advised and not log: logg.warn( 'Consider logarithmizing X with `scv.pp.log1p` for better results.' ) return adata if copy else None
def filter_genes_dispersion(data, flavor='seurat', min_disp=None, max_disp=None, min_mean=None, max_mean=None, n_bins=20, n_top_genes=None, log=True, copy=False): """Extract highly variable genes. The normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for genes falling into a given bin for mean expression of genes. This means that for each bin of mean expression, highly variable genes are selected. Parameters ---------- data : :class:`~anndata.AnnData`, `np.ndarray`, `sp.sparse` The (annotated) data matrix of shape `n_obs` × `n_vars`. Rows correspond to cells and columns to genes. flavor : {'seurat', 'cell_ranger', 'svr'}, optional (default: 'seurat') Choose the flavor for computing normalized dispersion. If choosing 'seurat', this expects non-logarithmized data - the logarithm of mean and dispersion is taken internally when `log` is at its default value `True`. For 'cell_ranger', this is usually called for logarithmized data - in this case you should set `log` to `False`. In their default workflows, Seurat passes the cutoffs whereas Cell Ranger passes `n_top_genes`. min_mean=0.0125, max_mean=3, min_disp=0.5, max_disp=`None` : `float`, optional If `n_top_genes` unequals `None`, these cutoffs for the means and the normalized dispersions are ignored. n_bins : `int` (default: 20) Number of bins for binning the mean gene expression. Normalization is done with respect to each bin. If just a single gene falls into a bin, the normalized dispersion is artificially set to 1. You'll be informed about this if you set `settings.verbosity = 4`. n_top_genes : `int` or `None` (default: `None`) Number of highly-variable genes to keep. log : `bool`, optional (default: `True`) Use the logarithm of the mean to variance ratio. copy : `bool`, optional (default: `False`) If an :class:`~anndata.AnnData` is passed, determines whether a copy is returned. Returns ------- If an AnnData `adata` is passed, returns or updates `adata` depending on \ `copy`. It filters the `adata` and adds the annotations """ adata = data.copy() if copy else data set_initial_size(adata) if n_top_genes is not None and adata.n_vars < n_top_genes: logg.info( 'Skip filtering by dispersion since number of variables are less than `n_top_genes`' ) else: if flavor is 'svr': mu = adata.X.mean(0).A1 if issparse(adata.X) else adata.X.mean(0) sigma = np.sqrt(adata.X.multiply(adata.X).mean(0).A1 - mu**2) if issparse(adata.X) else adata.X.std(0) log_mu = np.log2(mu) log_cv = np.log2(sigma / mu) from sklearn.svm import SVR clf = SVR(gamma=150. / len(mu)) clf.fit(log_mu[:, None], log_cv) score = log_cv - clf.predict(log_mu[:, None]) nth_score = np.sort(score)[::-1][n_top_genes] adata._inplace_subset_var(score >= nth_score) else: from scanpy.api.pp import filter_genes_dispersion filter_genes_dispersion(adata, flavor=flavor, min_disp=min_disp, max_disp=max_disp, min_mean=min_mean, max_mean=max_mean, n_bins=n_bins, n_top_genes=n_top_genes, log=log) return adata if copy else None
def FindHVG(self): self.scRNAseq_HVGData = filter_genes_dispersion( self.scRNAseq_Propcessed, copy=True)
def filter_and_normalize(data, min_counts=3, min_counts_u=3, min_cells=None, min_cells_u=None, n_top_genes=None, log=True, plot=False, copy=False): """Filtering, normalization and log transform Expects non-logarithmized data. If using logarithmized data, pass `log=False`. Runs the following steps .. code:: python sc.pp.filter_genes(adata, min_counts=10) sc.pp.normalize_per_cell(adata) sc.pp.filter_genes_dispersion(adata, n_top_genes=10000) sc.pp.normalize_per_cell(adata) if log: sc.pp.log1p(adata) Arguments --------- data: :class:`~anndata.AnnData` Annotated data matrix. min_counts: `int` (default: 10) Minimum number of gene counts per cell. n_top_genes: `int` (default: 10000) Number of genes to keep. log: `bool` (default: `True`) Take logarithm. copy: `bool` (default: `False`) Return a copy of `adata` instead of updating it. Returns ------- Returns or updates `adata` depending on `copy`. """ adata = data.copy() if copy else data from scanpy.api.pp import filter_genes, filter_genes_dispersion, normalize_per_cell, log1p def filter_genes_u(adata, min_counts_u=None, min_cells_u=None): counts = adata.layers[ 'unspliced'] if min_counts_u is not None else adata.layers[ 'unspliced'] > 0 counts = counts.sum(0).A1 if issparse(counts) else counts.sum(0) adata._inplace_subset_var(counts >= ( min_counts_u if min_counts_u is not None else min_cells_u)) if min_counts is not None: filter_genes(adata, min_counts=min_counts) if min_cells is not None: filter_genes(adata, min_cells=min_cells) if 'unspliced' in adata.layers.keys(): if min_counts_u is not None: filter_genes_u(adata, min_counts_u=min_counts_u) if min_cells_u is not None: filter_genes_u(adata, min_cells_u=min_cells_u) if n_top_genes is not None and n_top_genes < adata.shape[1]: normalize_per_cell(adata) filter_result = filter_genes_dispersion(adata.X, n_top_genes=n_top_genes, log=False) if plot: from scanpy.plotting.preprocessing import filter_genes_dispersion as plot_filter_genes_dispersion plot_filter_genes_dispersion(filter_result, log=True) adata._inplace_subset_var(filter_result.gene_subset) #filter_genes_dispersion(adata, n_top_genes=n_top_genes) normalize_per_cell(adata) if log: log1p(adata) return adata if copy else None
def recluster(adata, celltype, celltype_label='leiden', min_mean=0.0125, max_mean=4, min_disp=0.5, resolution=1.0, regress_out_key=None, random_seed=0, show_plot_filter=False, method='leiden'): """ Perform subclustering on specific celltype to identify subclusters. Extract all cells that belong to the pre-labeled celltype into a new data subset. This datasubset is initialized with the raw data contained in adata.raw. New highly variable genes are selected and a new clustering is performed. The function returns the adata subset with the new clustering annotation. This can be performed on leiden clusters by setting celltype_label = 'leiden' and passing the clusters that are to be selected for reclustering as strings or tuple of strings to the parameter celltype. Parameters ---------- adata: the complete AnnData object of the Dataset. celltype: `str` or (`str`) string identifying the cluster which is to be filtered out, if more than one is to be selected please pass them as a tuple not as a list! celltype_label: `str` | default = 'leiden' string identifying which column in adata.obs will be matching with the celltype argument. min_mean: `float` | default = 0.0125 the minimum gene expression a gene must have to be considered highly variable max_mean: `float` | default = 4 the maximum gene expression a gene can have to be considered highly variable min_disp: `float` | default = 0.5 the minimum dispersion a gene must have to be considered highly variable regress_out_key: `list of str` | default = None A list of string identifiers of the adata.obs columns that should be regressed out before performing clustering. If None then no regress_out is calculated. random_seed: `int` | default = 0 the random seed that is used to produce reproducible PCA, clustering and UMAP results show_plot_filter: `bool` | default = False boolian value indicating if a plot showing the filtering results for highly variable gene detection should be displayed or not method: `str` | default = 'louvain' clustering method to use for the reclustering of the datasubset. Possible:louvain/leiden Returns ------- AnnData object containing the subcluster annotated with PCA, nearest neighbors, louvain cluster, and UMAP coordinates. Examples -------- For a more detailed example of the entire reclustering process please refer to the code examples. >>> import besca as bc >>> import scanpy.api as sc >>> adata = bc.datasets.pbmc3k_processed() >>> adata_subset = bc.tl.rc.recluster(adata, celltype=('0', '1', '3', '6'), resolution = 1.3) >>> sc.pl.umap(adata_subset, color = ['louvain', 'CD3G', 'CD8A', 'CD4', 'IL7R', 'NKG7', 'GNLY']) """ if (not method in ['leiden', 'louvain']): raise ValueError("method argument should be leiden or louvain") if type(celltype) == str: cluster_subset = _subset_adata( adata, adata.obs.get(celltype_label) == celltype) elif type(celltype) == tuple: filter = adata.obs.get(celltype_label) == 'NONE' for i in range(len(celltype)): filter = filter | (adata.obs.get(celltype_label) == celltype[i]) cluster_subset = _subset_adata(adata, filter) else: sys.exit('specify cluster input as a string or tuple') cluster_subset.raw = cluster_subset #identify highly variable genes filter_result = filter_genes_dispersion(cluster_subset.X, min_mean=min_mean, max_mean=max_mean, min_disp=min_disp) if show_plot_filter: plot_filter(filter_result) print('In total', str(sum(filter_result.gene_subset)), 'highly variable genes selected within cluster') #apply filter cluster_subset = _subset_adata(cluster_subset, filter_result.gene_subset, axis=1, raw=False) #perform further processing log1p(cluster_subset) if regress_out_key is not None: regress_out(cluster_subset, keys=regress_out_key) sc_scale(cluster_subset) sc_pca( cluster_subset, random_state=random_seed, svd_solver='arpack' ) #using `svd_solver='arpack' ensures that the PCA leads to reproducible results neighbors(cluster_subset, n_neighbors=10, random_state=random_seed) umap(cluster_subset, random_state=random_seed) if method == 'louvain': louvain(cluster_subset, resolution=resolution, random_state=random_seed) if method == 'leiden': leiden(cluster_subset, resolution=resolution, random_state=random_seed) return (cluster_subset)
def filter_and_normalize(data, min_counts=None, min_counts_u=None, min_cells=None, min_cells_u=None, n_top_genes=None, flavor='seurat', log=True, copy=False): """Filtering, normalization and log transform Expects non-logarithmized data. If using logarithmized data, pass `log=False`. Runs the following steps .. code:: python scv.pp.filter_genes(adata) scv.pp.normalize_per_cell(adata) if n_top_genes is not None: scv.pp.filter_genes_dispersion(adata) if log: scv.pp.log1p(adata) Arguments --------- data: :class:`~anndata.AnnData` Annotated data matrix. min_counts: `int` (default: `None`) Minimum number of counts required for a gene to pass filtering (spliced). min_counts_u: `int` (default: `None`) Minimum number of counts required for a gene to pass filtering (unspliced). min_cells: `int` (default: `None`) Minimum number of cells expressed required for a gene to pass filtering (spliced). min_cells_u: `int` (default: `None`) Minimum number of cells expressed required for a gene to pass filtering (unspliced). n_top_genes: `int` (default: `None`) Number of genes to keep. flavor: {'seurat', 'cell_ranger', 'svr'}, optional (default: 'seurat') Choose the flavor for computing normalized dispersion. If choosing 'seurat', this expects non-logarithmized data. log: `bool` (default: `True`) Take logarithm. copy: `bool` (default: `False`) Return a copy of `adata` instead of updating it. Returns ------- Returns or updates `adata` depending on `copy`. """ adata = data.copy() if copy else data filter_genes(adata, min_counts=min_counts, min_counts_u=min_counts_u, min_cells=min_cells, min_cells_u=min_cells_u) normalize_per_cell(adata) if n_top_genes is not None: filter_genes_dispersion(adata, n_top_genes=n_top_genes, flavor=flavor) if log: log1p(adata) return adata if copy else None