def transform(self, X): """ Used to transform data after fiting :param X: list :return: list """ X = check_array(X, self.sparse_format, copy=self.copy, estimator='the normalize function', dtype=FLOAT_DTYPES) if self.axis == 0: X = X.T if sparse.issparse(X): if self.norm == 'l1': inplace_csr_row_normalize_l1(X) elif self.norm == 'l2': inplace_csr_row_normalize_l2(X) elif self.norm == 'max': norms_elementwise = self.norms.repeat(np.diff(X.indptr)) mask = norms_elementwise != 0 X.data[mask] /= norms_elementwise[mask] else: X /= self.norms[:, np.newaxis] if self.axis == 0: X = X.T return X
def fit(self, X): """ Used to fit Noramlizer with data :param X: list :return: nothing """ if self.norm not in ('l1', 'l2', 'max'): raise ValueError("'%s' is not a supported norm" % self.norm) if self.axis == 0: self.sparse_format = 'csc' elif self.axis == 1: self.sparse_format = 'csr' else: raise ValueError("'%d' is not a supported axis" % self.axis) X = check_array(X, self.sparse_format, copy=self.copy, estimator='the normalize function', dtype=FLOAT_DTYPES) if self.axis == 0: X = X.T if sparse.issparse(X): if self.norm == 'l1': inplace_csr_row_normalize_l1(X) elif self.norm == 'l2': inplace_csr_row_normalize_l2(X) elif self.norm == 'max': _, self.norms = min_max_axis(X, 1) else: if self.norm == 'l1': self.norms = np.abs(X).sum(axis=1) elif self.norm == 'l2': self.norms = row_norms(X) elif self.norm == 'max': self.norms = np.max(X, axis=1) self.norms = _handle_zeros_in_scale(self.norms, copy=False)
def normalize(X, norm='l2', axis=1, copy=True, return_norm=False, shrink=0): """Scale input vectors individually to unit norm (vector length). Read more in the :ref:`User Guide <preprocessing_normalization>`. Parameters ---------- X : {array-like, sparse matrix}, shape [n_samples, n_features] The data to normalize, element by element. scipy.sparse matrices should be in CSR format to avoid an un-necessary copy. norm : 'l1', 'l2', or 'max', optional ('l2' by default) The norm to use to normalize each non zero sample (or each non-zero feature if axis is 0). axis : 0 or 1, optional (1 by default) axis used to normalize the data along. If 1, independently normalize each sample, otherwise (if 0) normalize each feature. copy : boolean, optional, default True set to False to perform inplace row normalization and avoid a copy (if the input is already a numpy array or a scipy.sparse CSR matrix and if axis is 1). return_norm : boolean, default False whether to return the computed norms Returns ------- X : {array-like, sparse matrix}, shape [n_samples, n_features] Normalized input X. norms : array, shape [n_samples] if axis=1 else [n_features] An array of norms along given axis for X. When X is sparse, a NotImplementedError will be raised for norm 'l1' or 'l2'. See also -------- Normalizer: Performs normalization using the ``Transformer`` API (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`). Notes ----- For a comparison of the different scalers, transformers, and normalizers, see :ref:`examples/preprocessing/plot_all_scaling.py <sphx_glr_auto_examples_preprocessing_plot_all_scaling.py>`. """ if norm not in ('l1', 'l2', 'max'): raise ValueError("'%s' is not a supported norm" % norm) if axis == 0: sparse_format = 'csc' elif axis == 1: sparse_format = 'csr' else: raise ValueError("'%d' is not a supported axis" % axis) X = check_array(X, sparse_format, copy=copy, estimator='the normalize function', dtype=FLOAT_DTYPES) if axis == 0: X = X.T if sparse.issparse(X): if return_norm and norm in ('l1', 'l2'): raise NotImplementedError("return_norm=True is not implemented " "for sparse matrices with norm 'l1' " "or norm 'l2'") if norm == 'l1': inplace_csr_row_normalize_l1(X) elif norm == 'l2': inplace_csr_row_normalize_l2(X, shrink) elif norm == 'max': _, norms = min_max_axis(X, 1) norms_elementwise = norms.repeat(np.diff(X.indptr)) mask = norms_elementwise != 0 X.data[mask] /= norms_elementwise[mask] else: if norm == 'l1': norms = np.abs(X).sum(axis=1) elif norm == 'l2': norms = row_norms(X) elif norm == 'max': norms = np.max(X, axis=1) norms = _handle_zeros_in_scale(norms, copy=False) X /= norms[:, np.newaxis] if axis == 0: X = X.T if return_norm: return X, norms else: return X
def normalize(self, X, norm='l2', axis=1, copy=True): """Normalize a dataset along any axis Parameters ---------- X : array or scipy.sparse matrix with shape [n_samples, n_features] The data to normalize, element by element. scipy.sparse matrices should be in CSR format to avoid an un-necessary copy. norm : 'l1' or 'l2', optional ('l2' by default) The norm to use to normalize each non zero sample (or each non-zero feature if axis is 0). axis : 0 or 1, optional (1 by default) axis used to normalize the data along. If 1, independently normalize each sample, otherwise (if 0) normalize each feature. copy : boolean, optional, default is True set to False to perform inplace row normalization and avoid a copy (if the input is already a numpy array or a scipy.sparse CSR matrix and if axis is 1). See also -------- :class:`sklearn.preprocessing.Normalizer` to perform normalization using the ``Transformer`` API (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`) """ if norm not in ('l1', 'l2'): raise ValueError("'%s' is not a supported norm" % norm) if axis == 0: sparse_format = 'csc' elif axis == 1: sparse_format = 'csr' else: raise ValueError("'%d' is not a supported axis" % axis) X = check_arrays(X, sparse_format=sparse_format, copy=copy)[0] warn_if_not_float(X, 'The normalize function') if axis == 0: X = X.T if sparse.issparse(X): if norm == 'l1': inplace_csr_row_normalize_l1(X) elif norm == 'l2': inplace_csr_row_normalize_l2(X) else: if norm == 'l1': norms = np.abs(X).sum(axis=1) norms[norms == 0.0] = 1.0 elif norm == 'l2': norms = row_norms(X) norms[norms == 0.0] = 1.0 X /= norms[:, np.newaxis] if axis == 0: X = X.T return X
def _one_fit(self): if self.verbose: print("\nCreating synthetic doublets...") self._createDoublets() # Normalize combined augmented set if self.verbose: print("Normalizing...") if self.normalizer is not None: aug_counts = self.normalizer( sp_sparse.vstack((self._raw_counts, self._raw_synthetics))) else: # Follows doubletdetection.plot.normalize_counts, but uses memoized normed raw_counts synth_lib_size = np.sum(self._raw_synthetics, axis=1).A1 aug_lib_size = np.concatenate([self._lib_size, synth_lib_size]) normed_synths = self._raw_synthetics.copy() inplace_csr_row_normalize_l1(normed_synths) aug_counts = sp_sparse.vstack( (self._normed_raw_counts, normed_synths)) aug_counts = np.log(aug_counts.A * np.median(aug_lib_size) + 0.1) self._norm_counts = aug_counts[:self._num_cells] self._synthetics = aug_counts[self._num_cells:] aug_counts = anndata.AnnData(aug_counts) aug_counts.obs["n_counts"] = aug_lib_size if self.standard_scaling is True: sc.pp.scale(aug_counts, max_value=15) if self.verbose: print("Running PCA...") sc.tl.pca(aug_counts, n_comps=self.n_components, random_state=self.random_state) if self.verbose: print("Clustering augmented data set...\n") if self.use_phenograph: f = io.StringIO() with redirect_stdout(f): fullcommunities, _, _ = phenograph.cluster( aug_counts.obsm["X_pca"], **self.phenograph_parameters) out = f.getvalue() if self.verbose: print(out) else: sc.pp.neighbors( aug_counts, random_state=self.random_state, method="umap", n_neighbors=10, ) sc.tl.louvain(aug_counts, random_state=self.random_state, resolution=4, directed=False) fullcommunities = np.array(aug_counts.obs["louvain"], dtype=int) min_ID = min(fullcommunities) self.communities_ = fullcommunities[:self._num_cells] self.synth_communities_ = fullcommunities[self._num_cells:] community_sizes = [ np.count_nonzero(fullcommunities == i) for i in np.unique(fullcommunities) ] if self.verbose: print("Found clusters [{0}, ... {2}], with sizes: {1}\n".format( min(fullcommunities), community_sizes, max(fullcommunities))) # Count number of fake doublets in each community and assign score # Number of synth/orig cells in each cluster. synth_cells_per_comm = collections.Counter(self.synth_communities_) orig_cells_per_comm = collections.Counter(self.communities_) community_IDs = orig_cells_per_comm.keys() community_scores = { i: float(synth_cells_per_comm[i]) / (synth_cells_per_comm[i] + orig_cells_per_comm[i]) for i in community_IDs } scores = np.array([community_scores[i] for i in self.communities_]) community_log_p_values = { i: hypergeom.logsf( synth_cells_per_comm[i], aug_counts.shape[0], self._synthetics.shape[0], synth_cells_per_comm[i] + orig_cells_per_comm[i], ) for i in community_IDs } log_p_values = np.array( [community_log_p_values[i] for i in self.communities_]) if min_ID < 0: scores[self.communities_ == -1] = np.nan log_p_values[self.communities_ == -1] = np.nan return scores, log_p_values
def fit(self, raw_counts): """Fits the classifier on raw_counts. Args: raw_counts (array-like): Count matrix, oriented cells by genes. Sets: all_scores_, all_p_values_, all_log_p_values_, communities_, top_var_genes, parents, synth_communities Returns: The fitted classifier. """ raw_counts = check_array( raw_counts, accept_sparse="csr", force_all_finite=True, ensure_2d=True, dtype="float32", ) if sp_sparse.issparse(raw_counts) is not True: if self.verbose: print("Sparsifying matrix.") raw_counts = csr_matrix(raw_counts) if self.n_top_var_genes > 0: if self.n_top_var_genes < raw_counts.shape[1]: gene_variances = (np.array(raw_counts.power(2).mean(axis=0)) - (np.array(raw_counts.mean(axis=0)))**2)[0] top_var_indexes = np.argsort(gene_variances) self.top_var_genes_ = top_var_indexes[-self.n_top_var_genes:] # csc if faster for column indexing raw_counts = raw_counts.tocsc() raw_counts = raw_counts[:, self.top_var_genes_] raw_counts = raw_counts.tocsr() self._raw_counts = raw_counts (self._num_cells, self._num_genes) = self._raw_counts.shape if self.normalizer is None: # Memoize these; default normalizer treats these invariant for all synths self._lib_size = np.sum(raw_counts, axis=1).A1 self._normed_raw_counts = self._raw_counts.copy() inplace_csr_row_normalize_l1(self._normed_raw_counts) self.all_scores_ = np.zeros((self.n_iters, self._num_cells)) self.all_log_p_values_ = np.zeros((self.n_iters, self._num_cells)) all_communities = np.zeros((self.n_iters, self._num_cells)) all_parents = [] all_synth_communities = np.zeros( (self.n_iters, int(self.boost_rate * self._num_cells))) for i in tqdm(range(self.n_iters)): if self.verbose: print("Iteration {:3}/{}".format(i + 1, self.n_iters)) self.all_scores_[i], self.all_log_p_values_[i] = self._one_fit() all_communities[i] = self.communities_ all_parents.append(self.parents_) all_synth_communities[i] = self.synth_communities_ # Release unneeded large data vars del self._raw_counts del self._norm_counts del self._raw_synthetics del self._synthetics if self.normalizer is None: del self._normed_raw_counts del self._lib_size self.communities_ = all_communities self.parents_ = all_parents self.synth_communities_ = all_synth_communities self.all_p_values_ = np.exp(self.all_log_p_values_) return self
def normalize(X, norm='l2', axis=1, copy=True): """Scale input vectors individually to unit norm (vector length). Parameters ---------- X : array or scipy.sparse matrix with shape [n_samples, n_features] The data to normalize, element by element. scipy.sparse matrices should be in CSR format to avoid an un-necessary copy. norm : 'l1' or 'l2', optional ('l2' by default) The norm to use to normalize each non zero sample (or each non-zero feature if axis is 0). axis : 0 or 1, optional (1 by default) axis used to normalize the data along. If 1, independently normalize each sample, otherwise (if 0) normalize each feature. copy : boolean, optional, default True set to False to perform inplace row normalization and avoid a copy (if the input is already a numpy array or a scipy.sparse CSR matrix and if axis is 1). See also -------- :class:`sklearn.preprocessing.Normalizer` to perform normalization using the ``Transformer`` API (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`) """ if norm not in ('l1', 'l2'): raise ValueError("'%s' is not a supported norm" % norm) if axis == 0: sparse_format = 'csc' elif axis == 1: sparse_format = 'csr' else: raise ValueError("'%d' is not a supported axis" % axis) X = check_array(X, sparse_format, copy=copy) warn_if_not_float(X, 'The normalize function') if axis == 0: X = X.T if sparse.issparse(X): X = check_array(X, accept_sparse=sparse_format, dtype=np.float64) if norm == 'l1': inplace_csr_row_normalize_l1(X) elif norm == 'l2': inplace_csr_row_normalize_l2(X) else: if norm == 'l1': norms = np.abs(X).sum(axis=1) norms[norms == 0.0] = 1.0 elif norm == 'l2': norms = row_norms(X) norms[norms == 0.0] = 1.0 X /= norms[:, np.newaxis] if axis == 0: X = X.T return X