def __init__( self, model: Union[SCVI, SCANVI, TOTALVI], adata: anndata.AnnData, trainer: Optional['Trainer'] = None, cell_type_key: str = None, batch_key: str = None, ): self.outer_model = model self.model = model.model self.model.eval() if trainer is None: self.trainer = model.trainer else: self.trainer = trainer self.adata = adata self.modified = getattr(model.model, 'encode_covariates', True) self.annotated = type(model) is SCANVI self.predictions = None self.certainty = None self.prediction_names = None self.class_check = None self.post_adata_2 = None if trainer is not None: if self.trainer.use_cuda: self.device = torch.device('cuda') else: self.device = torch.device('cpu') else: self.device = next(self.model.parameters()).get_device() if issparse(self.adata.X): X = self.adata.X.toarray() else: X = self.adata.X self.x_tensor = torch.tensor(X, device=self.device) self.labels = None self.label_tensor = None if self.annotated: self.labels = get_from_registry(self.adata, "labels").astype(np.int8) self.label_tensor = torch.tensor(self.labels, device=self.device) self.cell_types = self.adata.obs[cell_type_key].tolist() self.batch_indices = get_from_registry(self.adata, "batch_indices").astype(np.int8) self.batch_tensor = torch.tensor(self.batch_indices, device=self.device) self.batch_names = self.adata.obs[batch_key].tolist() self.celltype_enc = [0] * len( self.adata.obs[cell_type_key].unique().tolist()) for i, cell_type in enumerate( self.adata.obs[cell_type_key].unique().tolist()): label = self.adata.obs['_scvi_labels'].unique().tolist()[i] self.celltype_enc[label] = cell_type self.post_adata = self.latent_as_anndata()
def _validate_anndata(self, adata: Optional[AnnData] = None, copy_if_view: bool = True): """Validate anndata has been properly registered, transfer if necessary.""" if adata is None: adata = self.adata if adata.is_view: if copy_if_view: logger.info("Received view of anndata, making copy.") adata = adata.copy() else: raise ValueError("Please run `adata = adata.copy()`") if "_scvi" not in adata.uns_keys(): logger.info("Input adata not setup with scvi. " + "attempting to transfer anndata setup") transfer_anndata_setup(self.scvi_setup_dict_, adata) is_nonneg_int = _check_nonnegative_integers( get_from_registry(adata, _CONSTANTS.X_KEY)) if not is_nonneg_int: logger.warning( "Make sure the registered X field in anndata contains unnormalized count data." ) _check_anndata_setup_equivalence(self.scvi_setup_dict_, adata) return adata
def scatac_raw_counts_properties( adata: anndata.AnnData, idx1: Union[List[int], np.ndarray], idx2: Union[List[int], np.ndarray], ) -> Dict[str, np.ndarray]: """ Computes and returns some statistics on the raw counts of two sub-populations. Parameters ---------- adata AnnData object setup with `scvi`. idx1 subset of indices describing the first population. idx2 subset of indices describing the second population. Returns ------- type Dict of ``np.ndarray`` containing, by pair (one for each sub-population). """ data = get_from_registry(adata, _CONSTANTS.X_KEY) data1 = data[idx1] data2 = data[idx2] mean1 = np.asarray((data1 > 0).mean(axis=0)).ravel() mean2 = np.asarray((data2 > 0).mean(axis=0)).ravel() properties = dict(emp_mean1=mean1, emp_mean2=mean2, emp_effect=(mean1 - mean2)) return properties
def _init_library_size( adata: anndata.AnnData, n_batch: dict ) -> Tuple[np.ndarray, np.ndarray]: """ Computes and returns library size. Parameters ---------- adata AnnData object setup with `scvi`. n_batch Number of batches. Returns ------- type Tuple of two 1 x n_batch ``np.ndarray`` containing the means and variances of library size in each batch in adata. If a certain batch is not present in the adata, the mean defaults to 0, and the variance defaults to 1. These defaults are arbitrary placeholders which should not be used in any downstream computation. """ data = get_from_registry(adata, _CONSTANTS.X_KEY) batch_indices = get_from_registry(adata, _CONSTANTS.BATCH_KEY) library_log_means = np.zeros(n_batch) library_log_vars = np.ones(n_batch) for i_batch in np.unique(batch_indices): idx_batch = np.squeeze(batch_indices == i_batch) batch_data = data[ idx_batch.nonzero()[0] ] # h5ad requires integer indexing arrays. sum_counts = batch_data.sum(axis=1) masked_log_sum = np.ma.log(sum_counts) if np.ma.is_masked(masked_log_sum): warnings.warn( "This dataset has some empty cells, this might fail inference." "Data should be filtered with `scanpy.pp.filter_cells()`" ) log_counts = masked_log_sum.filled(0) library_log_means[i_batch] = np.mean(log_counts).astype(np.float32) library_log_vars[i_batch] = np.var(log_counts).astype(np.float32) return library_log_means.reshape(1, -1), library_log_vars.reshape(1, -1)
def scrna_raw_counts_properties( adata: anndata.AnnData, idx1: Union[List[int], np.ndarray], idx2: Union[List[int], np.ndarray], ) -> Dict[str, np.ndarray]: """ Computes and returns some statistics on the raw counts of two sub-populations. Parameters ---------- adata AnnData object setup with `scvi`. idx1 subset of indices describing the first population. idx2 subset of indices describing the second population. Returns ------- type Dict of ``np.ndarray`` containing, by pair (one for each sub-population), mean expression per gene, proportion of non-zero expression per gene, mean of normalized expression. """ data = get_from_registry(adata, _CONSTANTS.X_KEY) data1 = data[idx1] data2 = data[idx2] mean1 = np.asarray((data1).mean(axis=0)).ravel() mean2 = np.asarray((data2).mean(axis=0)).ravel() nonz1 = np.asarray((data1 != 0).mean(axis=0)).ravel() nonz2 = np.asarray((data2 != 0).mean(axis=0)).ravel() key = "_scvi_raw_norm_scaling" if key not in adata.obs.keys(): scaling_factor = 1 / np.asarray(data.sum(axis=1)).ravel().reshape( -1, 1) scaling_factor *= 1e4 adata.obs[key] = scaling_factor.ravel() else: scaling_factor = adata.obs[key].to_numpy().ravel().reshape(-1, 1) if issubclass(type(data), sp_sparse.spmatrix): norm_data1 = data1.multiply(scaling_factor[idx1]) norm_data2 = data2.multiply(scaling_factor[idx2]) else: norm_data1 = data1 * scaling_factor[idx1] norm_data2 = data2 * scaling_factor[idx2] norm_mean1 = np.asarray(norm_data1.mean(axis=0)).ravel() norm_mean2 = np.asarray(norm_data2.mean(axis=0)).ravel() properties = dict( raw_mean1=mean1, raw_mean2=mean2, non_zeros_proportion1=nonz1, non_zeros_proportion2=nonz2, raw_normalized_mean1=norm_mean1, raw_normalized_mean2=norm_mean2, ) return properties
def cite_seq_raw_counts_properties( adata: anndata.AnnData, idx1: Union[List[int], np.ndarray], idx2: Union[List[int], np.ndarray], ) -> Dict[str, np.ndarray]: """ Computes and returns some statistics on the raw counts of two sub-populations. Parameters ---------- adata AnnData object setup with `scvi`. idx1 subset of indices describing the first population. idx2 subset of indices describing the second population. Returns ------- type Dict of ``np.ndarray`` containing, by pair (one for each sub-population), mean expression per gene, proportion of non-zero expression per gene, mean of normalized expression. """ gp = scrna_raw_counts_properties(adata, idx1, idx2) protein_exp = get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY) nan = np.array([np.nan] * len(adata.uns["_scvi"]["protein_names"])) protein_exp = get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY) mean1_pro = np.asarray(protein_exp[idx1].mean(0)) mean2_pro = np.asarray(protein_exp[idx2].mean(0)) nonz1_pro = np.asarray((protein_exp[idx1] > 0).mean(0)) nonz2_pro = np.asarray((protein_exp[idx2] > 0).mean(0)) properties = dict( raw_mean1=np.concatenate([gp["raw_mean1"], mean1_pro]), raw_mean2=np.concatenate([gp["raw_mean2"], mean2_pro]), non_zeros_proportion1=np.concatenate( [gp["non_zeros_proportion1"], nonz1_pro]), non_zeros_proportion2=np.concatenate( [gp["non_zeros_proportion2"], nonz2_pro]), raw_normalized_mean1=np.concatenate([gp["raw_normalized_mean1"], nan]), raw_normalized_mean2=np.concatenate([gp["raw_normalized_mean2"], nan]), ) return properties
def test_data_format(): # if data was dense np array, check after setup_anndata, data is C_CONTIGUOUS adata = synthetic_iid(run_setup_anndata=False) old_x = adata.X old_pro = adata.obsm["protein_expression"] old_obs = adata.obs adata.X = np.asfortranarray(old_x) adata.obsm["protein_expression"] = np.asfortranarray(old_pro) assert adata.X.flags["C_CONTIGUOUS"] is False assert adata.obsm["protein_expression"].flags["C_CONTIGUOUS"] is False _setup_anndata(adata, protein_expression_obsm_key="protein_expression") assert adata.X.flags["C_CONTIGUOUS"] is True assert adata.obsm["protein_expression"].flags["C_CONTIGUOUS"] is True assert np.array_equal(old_x, adata.X) assert np.array_equal(old_pro, adata.obsm["protein_expression"]) assert np.array_equal(old_obs, adata.obs) assert np.array_equal(adata.X, get_from_registry(adata, _CONSTANTS.X_KEY)) assert np.array_equal( adata.obsm["protein_expression"], get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY), ) # if obsm is dataframe, make it C_CONTIGUOUS if it isnt adata = synthetic_iid() pe = np.asfortranarray(adata.obsm["protein_expression"]) adata.obsm["protein_expression"] = pd.DataFrame(pe, index=adata.obs_names) assert adata.obsm["protein_expression"].to_numpy( ).flags["C_CONTIGUOUS"] is False _setup_anndata(adata, protein_expression_obsm_key="protein_expression") new_pe = get_from_registry(adata, "protein_expression") assert new_pe.to_numpy().flags["C_CONTIGUOUS"] is True assert np.array_equal(pe, new_pe) assert np.array_equal(adata.X, get_from_registry(adata, _CONSTANTS.X_KEY)) assert np.array_equal( adata.obsm["protein_expression"], get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY), )
def _validate_anndata( self, adata: Optional[AnnData] = None, copy_if_view: bool = True ): adata = super()._validate_anndata(adata, copy_if_view) error_msg = "Number of {} in anndata different from when setup_anndata was run. Please rerun setup_anndata." if _CONSTANTS.PROTEIN_EXP_KEY in adata.uns["_scvi"]["data_registry"].keys(): if ( self.summary_stats["n_proteins"] != get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY).shape[1] ): raise ValueError(error_msg.format("proteins")) else: raise ValueError("No protein data found, please setup or transfer anndata") return adata
def create_doublets( adata: AnnData, doublet_ratio: int, indices: Optional[Sequence[int]] = None, seed: int = 1, ) -> AnnData: """Simulate doublets. Parameters ---------- adata AnnData object setup with :func:`~scvi.data.setup_anndata`. doublet_ratio Ratio of generated doublets to produce relative to number of cells in adata or length of indices, if not `None`. indices Indices of cells in adata to use. If `None`, all cells are used. seed Seed for reproducibility """ n_obs = adata.n_obs if indices is None else len(indices) num_doublets = doublet_ratio * n_obs # counts can be in many locations, this uses where it was registered in setup x = get_from_registry(adata, _CONSTANTS.X_KEY) if indices is not None: x = x[indices] random_state = np.random.RandomState(seed=seed) parent_inds = random_state.choice(n_obs, size=(num_doublets, 2)) doublets = x[parent_inds[:, 0]] + x[parent_inds[:, 1]] doublets_ad = AnnData(doublets) doublets_ad.var_names = adata.var_names doublets_ad.obs_names = [ "sim_doublet_{}".format(i) for i in range(num_doublets) ] # if adata setup with a layer, need to add layer to doublets adata data_registry = adata.uns["_scvi"]["data_registry"] x_loc = data_registry[_CONSTANTS.X_KEY]["attr_name"] layer = (data_registry[_CONSTANTS.X_KEY]["attr_key"] if x_loc == "layers" else None) if layer is not None: doublets_ad.layers[layer] = doublets return doublets_ad
def _validate_anndata( self, adata: Optional[AnnData] = None, copy_if_view: bool = True ): adata = super()._validate_anndata(adata, copy_if_view) error_msg = "Number of {} in anndata different from when setup_anndata was run. Please rerun setup_anndata." if _CONSTANTS.PROTEIN_EXP_KEY in adata.uns["_scvi"]["data_registry"].keys(): pro_exp = get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY) if self.summary_stats["n_proteins"] != pro_exp.shape[1]: raise ValueError(error_msg.format("proteins")) is_nonneg_int = _check_nonnegative_integers(pro_exp) if not is_nonneg_int: warnings.warn( "Make sure the registered protein expression in anndata contains unnormalized count data." ) else: raise ValueError("No protein data found, please setup or transfer anndata") return adata
def create_doublets(adata: AnnData, seed: int = 1, doublet_ratio: int = 2) -> AnnData: """Simulate doublets.""" num_doublets = doublet_ratio * adata.n_obs # counts can be in many locations, this uses where it was registered in setup x = get_from_registry(adata, _CONSTANTS.X_KEY) # TODO: needs a random state so it's reproducible parent_inds = np.random.choice(adata.n_obs, size=(num_doublets, 2)) doublets = x[parent_inds[:, 0]] + x[parent_inds[:, 1]] doublets_ad = AnnData(doublets) doublets_ad.var_names = adata.var_names doublets_ad.obs_names = [ "sim_doublet_{}".format(i) for i in range(num_doublets) ] return doublets_ad
def scale_sampler( self, selection: Union[List[bool], np.ndarray], n_samples: Optional[int] = 5000, n_samples_per_cell: Optional[int] = None, batchid: Optional[Union[List[int], np.ndarray]] = None, use_observed_batches: Optional[bool] = False, give_mean: Optional[bool] = False, ) -> dict: """ Samples the posterior scale using the variational posterior distribution. Parameters ---------- selection Mask or list of cell ids to select n_samples Number of samples in total per batch (fill either `n_samples_total` or `n_samples_per_cell`) n_samples_per_cell Number of time we sample from each observation per batch (fill either `n_samples_total` or `n_samples_per_cell`) batchid Biological batch for which to sample from. Default (None) sample from all batches use_observed_batches Whether normalized means are conditioned on observed batches or if observed batches are to be used give_mean Return mean of values Returns ------- type Dictionary containing: `scale` Posterior aggregated scale samples of shape (n_samples, n_genes) where n_samples correspond to either: - n_bio_batches * n_cells * n_samples_per_cell or - n_samples_total `batch` associated batch ids """ # Get overall number of desired samples and desired batches if batchid is None and not use_observed_batches: # TODO determine if we iterate over all categorical batches from train dataset # or just the batches in adata batchid = np.unique( get_from_registry(self.adata, key=_CONSTANTS.BATCH_KEY)) if use_observed_batches: if batchid is not None: raise ValueError("Unconsistent batch policy") batchid = [None] if n_samples is None and n_samples_per_cell is None: n_samples = 5000 elif n_samples_per_cell is not None and n_samples is None: n_samples = n_samples_per_cell * len(selection) if (n_samples_per_cell is not None) and (n_samples is not None): warnings.warn( "n_samples and n_samples_per_cell were provided. Ignoring n_samples_per_cell" ) n_samples = int(n_samples / len(batchid)) if n_samples == 0: warnings.warn( "very small sample size, please consider increasing `n_samples`" ) n_samples = 2 # Selection of desired cells for sampling if selection is None: raise ValueError( "selections should be a list of cell subsets indices") selection = np.asarray(selection) if selection.dtype is np.dtype("bool"): if len(selection) < self.adata.shape[0]: raise ValueError("Mask must be same length as adata.") selection = np.asarray(np.where(selection)[0].ravel()) # Sampling loop px_scales = [] batch_ids = [] for batch_idx in batchid: idx = np.random.choice( np.arange(self.adata.shape[0])[selection], n_samples) px_scales.append( self.model_fn(self.adata, indices=idx, transform_batch=batch_idx)) batch_idx = batch_idx if batch_idx is not None else np.nan batch_ids.append([batch_idx] * px_scales[-1].shape[0]) px_scales = np.concatenate(px_scales) batch_ids = np.concatenate(batch_ids).reshape(-1) if px_scales.shape[0] != batch_ids.shape[0]: raise ValueError( "sampled scales and batches have inconsistent shapes") if give_mean: px_scales = px_scales.mean(0) return dict(scale=px_scales, batch=batch_ids)
def _get_totalvi_protein_priors(adata, n_cells=100): """Compute an empirical prior for protein background.""" import warnings from sklearn.exceptions import ConvergenceWarning from sklearn.mixture import GaussianMixture warnings.filterwarnings("error") batch = get_from_registry(adata, _CONSTANTS.BATCH_KEY).ravel() cats = adata.uns["_scvi"]["categorical_mappings"]["_scvi_batch"]["mapping"] codes = np.arange(len(cats)) batch_avg_mus, batch_avg_scales = [], [] for b in np.unique(codes): # can happen during online updates # the values of these batches will not be used num_in_batch = np.sum(batch == b) if num_in_batch == 0: batch_avg_mus.append(0) batch_avg_scales.append(1) continue pro_exp = get_from_registry(adata, _CONSTANTS.PROTEIN_EXP_KEY)[batch == b] # for missing batches, put dummy values -- scarches case, will be replaced anyway if pro_exp.shape[0] == 0: batch_avg_mus.append(0.0) batch_avg_scales.append(0.05) cells = np.random.choice(np.arange(pro_exp.shape[0]), size=n_cells) if isinstance(pro_exp, pd.DataFrame): pro_exp = pro_exp.to_numpy() pro_exp = pro_exp[cells] gmm = GaussianMixture(n_components=2) mus, scales = [], [] # fit per cell GMM for c in pro_exp: try: gmm.fit(np.log1p(c.reshape(-1, 1))) # when cell is all 0 except ConvergenceWarning: mus.append(0) scales.append(0.05) continue means = gmm.means_.ravel() sorted_fg_bg = np.argsort(means) mu = means[sorted_fg_bg].ravel()[0] covariances = gmm.covariances_[sorted_fg_bg].ravel()[0] scale = np.sqrt(covariances) mus.append(mu) scales.append(scale) # average distribution over cells batch_avg_mu = np.mean(mus) batch_avg_scale = np.sqrt(np.sum(np.square(scales)) / (n_cells ** 2)) batch_avg_mus.append(batch_avg_mu) batch_avg_scales.append(batch_avg_scale) # repeat prior for each protein batch_avg_mus = np.array(batch_avg_mus, dtype=np.float32).reshape(1, -1) batch_avg_scales = np.array(batch_avg_scales, dtype=np.float32).reshape(1, -1) batch_avg_mus = np.tile(batch_avg_mus, (pro_exp.shape[1], 1)) batch_avg_scales = np.tile(batch_avg_scales, (pro_exp.shape[1], 1)) warnings.resetwarnings() return batch_avg_mus, batch_avg_scales
def test_setup_anndata(): # test regular setup adata = synthetic_iid(run_setup_anndata=False) _setup_anndata( adata, batch_key="batch", labels_key="labels", protein_expression_obsm_key="protein_expression", protein_names_uns_key="protein_names", ) np.testing.assert_array_equal( get_from_registry(adata, "batch_indices"), np.array(adata.obs["_scvi_batch"]).reshape((-1, 1)), ) np.testing.assert_array_equal( get_from_registry(adata, "labels"), np.array(adata.obs["labels"].cat.codes).reshape((-1, 1)), ) np.testing.assert_array_equal(get_from_registry(adata, "X"), adata.X) np.testing.assert_array_equal( get_from_registry(adata, "protein_expression"), adata.obsm["protein_expression"], ) np.testing.assert_array_equal(adata.uns["_scvi"]["protein_names"], adata.uns["protein_names"]) # test that error is thrown if its a view: adata = synthetic_iid() with pytest.raises(ValueError): _setup_anndata(adata[1]) # If obsm is a df and protein_names_uns_key is None, protein names should be grabbed from column of df adata = synthetic_iid() new_protein_names = np.array(random.sample(range(100), 100)).astype("str") df = pd.DataFrame( adata.obsm["protein_expression"], index=adata.obs_names, columns=new_protein_names, ) adata.obsm["protein_expression"] = df _setup_anndata(adata, protein_expression_obsm_key="protein_expression") np.testing.assert_array_equal(adata.uns["_scvi"]["protein_names"], new_protein_names) # test that layer is working properly adata = synthetic_iid() true_x = adata.X adata.layers["X"] = true_x adata.X = np.ones_like(adata.X) _setup_anndata(adata, layer="X") np.testing.assert_array_equal(get_from_registry(adata, "X"), true_x) # test that it creates layers and batch if no layers_key is passed adata = synthetic_iid() _setup_anndata( adata, protein_expression_obsm_key="protein_expression", protein_names_uns_key="protein_names", ) np.testing.assert_array_equal(get_from_registry(adata, "batch_indices"), np.zeros((adata.shape[0], 1))) np.testing.assert_array_equal(get_from_registry(adata, "labels"), np.zeros((adata.shape[0], 1)))
def get_bayes_factors( self, idx1: Union[List[bool], np.ndarray], idx2: Union[List[bool], np.ndarray], mode: Literal["vanilla", "change"] = "vanilla", batchid1: Optional[Sequence[Union[Number, str]]] = None, batchid2: Optional[Sequence[Union[Number, str]]] = None, use_observed_batches: Optional[bool] = False, n_samples: int = 5000, use_permutation: bool = False, m_permutation: int = 10000, change_fn: Optional[Union[str, Callable]] = None, m1_domain_fn: Optional[Callable] = None, delta: Optional[float] = 0.5, pseudocounts: Union[float, None] = 0.0, cred_interval_lvls: Optional[Union[List[float], np.ndarray]] = None, ) -> Dict[str, np.ndarray]: r""" A unified method for differential expression inference. Two modes coexist: - the `"vanilla"` mode follows protocol described in [Lopez18]_ and [Xu21]_ In this case, we perform hypothesis testing based on the hypotheses .. math:: M_1: h_1 > h_2 ~\text{and}~ M_2: h_1 \leq h_2. DE can then be based on the study of the Bayes factors .. math:: \log p(M_1 | x_1, x_2) / p(M_2 | x_1, x_2). - the `"change"` mode (described in [Boyeau19]_) This mode consists of estimating an effect size random variable (e.g., log fold-change) and performing Bayesian hypothesis testing on this variable. The `change_fn` function computes the effect size variable :math:`r` based on two inputs corresponding to the posterior quantities (e.g., normalized expression) in both populations. Hypotheses: .. math:: M_1: r \in R_1 ~\text{(effect size r in region inducing differential expression)} .. math:: M_2: r \notin R_1 ~\text{(no differential expression)} To characterize the region :math:`R_1`, which induces DE, the user has two choices. 1. A common case is when the region :math:`[-\delta, \delta]` does not induce differential expression. If the user specifies a threshold delta, we suppose that :math:`R_1 = \mathbb{R} \setminus [-\delta, \delta]` 2. Specify an specific indicator function: .. math:: f: \mathbb{R} \mapsto \{0, 1\} ~\text{s.t.}~ r \in R_1 ~\text{iff.}~ f(r) = 1. Decision-making can then be based on the estimates of .. math:: p(M_1 \mid x_1, x_2). Both modes require to sample the posterior distributions. To that purpose, we sample the posterior in the following way: 1. The posterior is sampled `n_samples` times for each subpopulation. 2. For computational efficiency (posterior sampling is quite expensive), instead of comparing the obtained samples element-wise, we can permute posterior samples. Remember that computing the Bayes Factor requires sampling :math:`q(z_A \mid x_A)` and :math:`q(z_B \mid x_B)`. Currently, the code covers several batch handling configurations: 1. If ``use_observed_batches=True``, then batch are considered as observations and cells' normalized means are conditioned on real batch observations. 2. If case (cell group 1) and control (cell group 2) are conditioned on the same batch ids. This requires ``set(batchid1) == set(batchid2)`` or ``batchid1 == batchid2 === None``. 3. If case and control are conditioned on different batch ids that do not intersect i.e., ``set(batchid1) != set(batchid2)`` and ``len(set(batchid1).intersection(set(batchid2))) == 0``. This function does not cover other cases yet and will warn users in such cases. Parameters ---------- mode one of ["vanilla", "change"] idx1 bool array masking subpopulation cells 1. Should be True where cell is from associated population idx2 bool array masking subpopulation cells 2. Should be True where cell is from associated population batchid1 List of batch ids for which you want to perform DE Analysis for subpopulation 1. By default, all ids are taken into account batchid2 List of batch ids for which you want to perform DE Analysis for subpopulation 2. By default, all ids are taken into account use_observed_batches Whether posterior values are conditioned on observed batches n_samples Number of posterior samples use_permutation Activates step 2 described above. Simply formulated, pairs obtained from posterior sampling will be randomly permuted so that the number of pairs used to compute Bayes Factors becomes `m_permutation`. m_permutation Number of times we will "mix" posterior samples in step 2. Only makes sense when `use_permutation=True` change_fn function computing effect size based on both posterior values m1_domain_fn custom indicator function of effect size regions inducing differential expression delta specific case of region inducing differential expression. In this case, we suppose that :math:`R \setminus [-\delta, \delta]` does not induce differential expression (LFC case). If the provided value is `None`, then a proper threshold is determined from the distribution of LFCs accross genes. pseudocounts pseudocount offset used for the mode `change`. When None, observations from non-expressed genes are used to estimate its value. cred_interval_lvls List of credible interval levels to compute for the posterior LFC distribution Returns ------- Differential expression properties """ # if not np.array_equal(self.indices, np.arange(len(self.dataset))): # warnings.warn( # "Differential expression requires a Posterior object created with all indices." # ) eps = 1e-8 # Normalized means sampling for both populations scales_batches_1 = self.scale_sampler( selection=idx1, batchid=batchid1, use_observed_batches=use_observed_batches, n_samples=n_samples, ) scales_batches_2 = self.scale_sampler( selection=idx2, batchid=batchid2, use_observed_batches=use_observed_batches, n_samples=n_samples, ) px_scale_mean1 = scales_batches_1["scale"].mean(axis=0) px_scale_mean2 = scales_batches_2["scale"].mean(axis=0) # Sampling pairs # The objective of code section below is to ensure than the samples of normalized # means we consider are conditioned on the same batch id batchid1_vals = np.unique(scales_batches_1["batch"]) batchid2_vals = np.unique(scales_batches_2["batch"]) create_pairs_from_same_batches = ( set(batchid1_vals) == set(batchid2_vals)) and not use_observed_batches if create_pairs_from_same_batches: # First case: same batch normalization in two groups logger.debug("Same batches in both cell groups") n_batches = len(set(batchid1_vals)) n_samples_per_batch = (m_permutation // n_batches if m_permutation is not None else None) logger.debug("Using {} samples per batch for pair matching".format( n_samples_per_batch)) scales_1 = [] scales_2 = [] for batch_val in set(batchid1_vals): # Select scale samples that originate from the same batch id scales_1_batch = scales_batches_1["scale"][ scales_batches_1["batch"] == batch_val] scales_2_batch = scales_batches_2["scale"][ scales_batches_2["batch"] == batch_val] # Create more pairs scales_1_local, scales_2_local = pairs_sampler( scales_1_batch, scales_2_batch, use_permutation=use_permutation, m_permutation=n_samples_per_batch, ) scales_1.append(scales_1_local) scales_2.append(scales_2_local) scales_1 = np.concatenate(scales_1, axis=0) scales_2 = np.concatenate(scales_2, axis=0) else: logger.debug("Ignoring batch conditionings to compare means") if len(set(batchid1_vals).intersection(set(batchid2_vals))) >= 1: warnings.warn( "Batchids of cells groups 1 and 2 are different but have an non-null " "intersection. Specific handling of such situations is not implemented " "yet and batch correction is not trustworthy.") scales_1, scales_2 = pairs_sampler( scales_batches_1["scale"], scales_batches_2["scale"], use_permutation=use_permutation, m_permutation=m_permutation, ) # Adding pseudocounts to the scales if pseudocounts is None: logger.debug("Estimating pseudocounts offet from the data") x = get_from_registry(self.adata, _CONSTANTS.X_KEY) where_zero_a = densify(np.max(x[idx1], 0)) == 0 where_zero_b = densify(np.max(x[idx2], 0)) == 0 pseudocounts = estimate_pseudocounts_offset( scales_a=scales_1, scales_b=scales_2, where_zero_a=where_zero_a, where_zero_b=where_zero_b, ) logger.debug("Using pseudocounts ~ {}".format(pseudocounts)) # Core of function: hypotheses testing based on the posterior samples we obtained above if mode == "vanilla": logger.debug("Differential expression using vanilla mode") proba_m1 = np.mean(scales_1 > scales_2, 0) proba_m2 = 1.0 - proba_m1 res = dict( proba_m1=proba_m1, proba_m2=proba_m2, bayes_factor=np.log(proba_m1 + eps) - np.log(proba_m2 + eps), scale1=px_scale_mean1, scale2=px_scale_mean2, ) elif mode == "change": logger.debug("Differential expression using change mode") # step 1: Construct the change function def lfc(x, y): return np.log2(x + pseudocounts) - np.log2(y + pseudocounts) if change_fn == "log-fold" or change_fn is None: change_fn = lfc elif not callable(change_fn): raise ValueError("'change_fn' attribute not understood") # step2: Construct the DE area function if m1_domain_fn is None: def m1_domain_fn(samples): delta_ = (delta if delta is not None else estimate_delta( lfc_means=samples.mean(0))) logger.debug("Using delta ~ {:.2f}".format(delta_)) return np.abs(samples) >= delta_ change_fn_specs = inspect.getfullargspec(change_fn) domain_fn_specs = inspect.getfullargspec(m1_domain_fn) if (len(change_fn_specs.args) != 2) | (len(domain_fn_specs.args) != 1): raise ValueError( "change_fn should take exactly two parameters as inputs; m1_domain_fn one parameter." ) try: change_distribution = change_fn(scales_1, scales_2) is_de = m1_domain_fn(change_distribution) delta_ = (estimate_delta(lfc_means=change_distribution.mean(0)) if delta is None else delta) except TypeError: raise TypeError( "change_fn or m1_domain_fn have has wrong properties." "Please ensure that these functions have the right signatures and" "outputs and that they can process numpy arrays") proba_m1 = np.mean(is_de, 0) change_distribution_props = describe_continuous_distrib( samples=change_distribution, credible_intervals_levels=cred_interval_lvls, ) change_distribution_props = { "lfc_" + key: val for (key, val) in change_distribution_props.items() } res = dict( proba_de=proba_m1, proba_not_de=1.0 - proba_m1, bayes_factor=np.log(proba_m1 + eps) - np.log(1.0 - proba_m1 + eps), scale1=px_scale_mean1, scale2=px_scale_mean2, pseudocounts=pseudocounts, delta=delta_, **change_distribution_props, ) else: raise NotImplementedError( "Mode {mode} not recognized".format(mode=mode)) return res