def dca_impute(self, data):
from dca.api import dca
import scanpy as sc
data = self.load()
adata = sc.AnnData(data.values, obs=data.index, var=data.columns)
dca(adata, threads=self.ncores)
return pd.DataFrame(adata.X)
def DCATransform(sc_data_matrix):
# Create a scanpy AnnData object
sc_data_matrix = sc.AnnData(numpy.transpose(sc_data_matrix.values))
# Filter genes with count<2
sc.pp.filter_genes(data=sc_data_matrix, min_counts=1)
# Apply DCA transform
dca(adata=sc_data_matrix, threads=4, epochs=10)
print("DCA Denoised data prepared")
return numpy.transpose(sc_data_matrix.X)
def dca(
adata,
mode='denoise',
ae_type='zinb-conddisp',
normalize_per_cell=True,
scale=True,
log1p=True,
# network args
hidden_size=(64, 32, 64),
hidden_dropout=0.,
batchnorm=True,
activation='relu',
init='glorot_uniform',
network_kwds={},
# training args
epochs=300,
reduce_lr=10,
early_stop=15,
batch_size=32,
optimizer='rmsprop',
random_state=0,
threads=None,
verbose=False,
training_kwds={},
return_model=False,
return_info=False,
copy=False):
"""Deep count autoencoder [Eraslan18]_.
Fits a count autoencoder to the raw count data given in the anndata object
in order to denoise the data and to capture hidden representation of
cells in low dimensions. Type of the autoencoder and return values are
determined by the parameters.
.. note::
More information and bug reports `here <https://github.com/theislab/dca>`__.
Parameters
----------
adata : :class:`~anndata.AnnData`
An anndata file with `.raw` attribute representing raw counts.
mode : `str`, optional. `denoise`(default), or `latent`.
`denoise` overwrites `adata.X` with denoised expression values.
In `latent` mode DCA adds `adata.obsm['X_dca']` to given adata
object. This matrix represent latent representation of cells via DCA.
ae_type : `str`, optional. `zinb-conddisp`(default), `zinb`, `nb-conddisp` or `nb`.
Type of the autoencoder. Return values and the architecture is
determined by the type e.g. `nb` does not provide dropout
probabilities. Types that end with "-conddisp", assumes that dispersion is mean dependant.
normalize_per_cell : `bool`, optional. Default: `True`.
If true, library size normalization is performed using
the `sc.pp.normalize_per_cell` function in Scanpy and saved into adata
object. Mean layer is re-introduces library size differences by
scaling the mean value of each cell in the output layer. See the
manuscript for more details.
scale : `bool`, optional. Default: `True`.
If true, the input of the autoencoder is centered using
`sc.pp.scale` function of Scanpy. Note that the output is kept as raw
counts as loss functions are designed for the count data.
log1p : `bool`, optional. Default: `True`.
If true, the input of the autoencoder is log transformed with a
pseudocount of one using `sc.pp.log1p` function of Scanpy.
hidden_size : `tuple` or `list`, optional. Default: (64, 32, 64).
Width of hidden layers.
hidden_dropout : `float`, `tuple` or `list`, optional. Default: 0.0.
Probability of weight dropout in the autoencoder (per layer if list
or tuple).
batchnorm : `bool`, optional. Default: `True`.
If true, batch normalization is performed.
activation : `str`, optional. Default: `relu`.
Activation function of hidden layers.
init : `str`, optional. Default: `glorot_uniform`.
Initialization method used to initialize weights.
network_kwds : `dict`, optional.
Additional keyword arguments for the autoencoder.
epochs : `int`, optional. Default: 300.
Number of total epochs in training.
reduce_lr : `int`, optional. Default: 10.
Reduces learning rate if validation loss does not improve in given number of epochs.
early_stop : `int`, optional. Default: 15.
Stops training if validation loss does not improve in given number of epochs.
batch_size : `int`, optional. Default: 32.
Number of samples in the batch used for SGD.
optimizer : `str`, optional. Default: "rmsprop".
Type of optimization method used for training.
random_state : `int`, optional. Default: 0.
Seed for python, numpy and tensorflow.
threads : `int` or None, optional. Default: None
Number of threads to use in training. All cores are used by default.
verbose : `bool`, optional. Default: `False`.
If true, prints additional information about training and architecture.
training_kwds : `dict`, optional.
Additional keyword arguments for the training process.
return_model : `bool`, optional. Default: `False`.
If true, trained autoencoder object is returned. See "Returns".
return_info : `bool`, optional. Default: `False`.
If true, all additional parameters of DCA are stored in `adata.obsm` such as dropout
probabilities (obsm['X_dca_dropout']) and estimated dispersion values
(obsm['X_dca_dispersion']), in case that autoencoder is of type
zinb or zinb-conddisp.
copy : `bool`, optional. Default: `False`.
If true, a copy of anndata is returned.
Returns
-------
If `copy` is true and `return_model` is false, AnnData object is returned.
In "denoise" mode, `adata.X` is overwritten with the denoised values. In "latent" mode, latent\
low dimensional representation of cells are stored in `adata.obsm['X_dca']` and `adata.X`\
is not modified. Note that these values are not corrected for library size effects.
If `return_info` is true, all estimated distribution parameters are stored in AnnData such as:
- `.obsm["X_dca_dropout"]` which is the mixture coefficient (pi) of the zero component\
in ZINB, i.e. dropout probability (only if `ae_type` is `zinb` or `zinb-conddisp`).
- `.obsm["X_dca_dispersion"]` which is the dispersion parameter of NB.
- `.uns["dca_loss_history"]` which stores the loss history of the training. See `.history`\
attribute of Keras History class for mode details.
Finally, the raw counts are stored in `.raw` attribute of AnnData object.
If `return_model` is given, trained model is returned. When both `copy` and `return_model`\
are true, a tuple of anndata and model is returned in that order.
"""
try:
from dca.api import dca
except ImportError:
raise ImportError(
'Please install dca package (>= 0.2.1) via `pip install dca`')
return dca(adata,
mode=mode,
ae_type=ae_type,
normalize_per_cell=normalize_per_cell,
scale=scale,
log1p=log1p,
hidden_size=hidden_size,
hidden_dropout=hidden_dropout,
batchnorm=batchnorm,
activation=activation,
init=init,
network_kwds=network_kwds,
epochs=epochs,
reduce_lr=reduce_lr,
early_stop=early_stop,
batch_size=batch_size,
optimizer=optimizer,
random_state=random_state,
threads=threads,
verbose=verbose,
training_kwds=training_kwds,
return_model=return_model)
def dca(
adata: AnnData,
mode: Literal['denoise', 'latent'] = 'denoise',
ae_type: _AEType = 'zinb-conddisp',
normalize_per_cell: bool = True,
scale: bool = True,
log1p: bool = True,
# network args
hidden_size: Sequence[int] = (64, 32, 64),
hidden_dropout: Union[float, Sequence[float]] = 0.0,
batchnorm: bool = True,
activation: str = 'relu',
init: str = 'glorot_uniform',
network_kwds: Mapping[str, Any] = MappingProxyType({}),
# training args
epochs: int = 300,
reduce_lr: int = 10,
early_stop: int = 15,
batch_size: int = 32,
optimizer: str = 'rmsprop',
random_state: Union[int, RandomState] = 0,
threads: Optional[int] = None,
learning_rate: Optional[float] = None,
verbose: bool = False,
training_kwds: Mapping[str, Any] = MappingProxyType({}),
return_model: bool = False,
return_info: bool = False,
copy: bool = False,
) -> Optional[AnnData]:
"""\
Deep count autoencoder [Eraslan18]_.
Fits a count autoencoder to the raw count data given in the anndata object
in order to denoise the data and to capture hidden representation of
cells in low dimensions. Type of the autoencoder and return values are
determined by the parameters.
.. note::
More information and bug reports `here <https://github.com/theislab/dca>`__.
Parameters
----------
adata
An anndata file with `.raw` attribute representing raw counts.
mode
`denoise` overwrites `adata.X` with denoised expression values.
In `latent` mode DCA adds `adata.obsm['X_dca']` to given adata
object. This matrix represent latent representation of cells via DCA.
ae_type
Type of the autoencoder. Return values and the architecture is
determined by the type e.g. `nb` does not provide dropout
probabilities. Types that end with "-conddisp", assumes that dispersion is mean dependant.
normalize_per_cell
If true, library size normalization is performed using
the `sc.pp.normalize_per_cell` function in Scanpy and saved into adata
object. Mean layer is re-introduces library size differences by
scaling the mean value of each cell in the output layer. See the
manuscript for more details.
scale
If true, the input of the autoencoder is centered using
`sc.pp.scale` function of Scanpy. Note that the output is kept as raw
counts as loss functions are designed for the count data.
log1p
If true, the input of the autoencoder is log transformed with a
pseudocount of one using `sc.pp.log1p` function of Scanpy.
hidden_size
Width of hidden layers.
hidden_dropout
Probability of weight dropout in the autoencoder (per layer if list
or tuple).
batchnorm
If true, batch normalization is performed.
activation
Activation function of hidden layers.
init
Initialization method used to initialize weights.
network_kwds
Additional keyword arguments for the autoencoder.
epochs
Number of total epochs in training.
reduce_lr
Reduces learning rate if validation loss does not improve in given number of epochs.
early_stop
Stops training if validation loss does not improve in given number of epochs.
batch_size
Number of samples in the batch used for SGD.
optimizer
Type of optimization method used for training.
random_state
Seed for python, numpy and tensorflow.
threads
Number of threads to use in training. All cores are used by default.
learning_rate
Learning rate to use in the training.
verbose
If true, prints additional information about training and architecture.
training_kwds
Additional keyword arguments for the training process.
return_model
If true, trained autoencoder object is returned. See "Returns".
return_info
If true, all additional parameters of DCA are stored in `adata.obsm` such as dropout
probabilities (obsm['X_dca_dropout']) and estimated dispersion values
(obsm['X_dca_dispersion']), in case that autoencoder is of type
zinb or zinb-conddisp.
copy
If true, a copy of anndata is returned.
Returns
-------
If `copy` is true and `return_model` is false, AnnData object is returned.
In "denoise" mode, `adata.X` is overwritten with the denoised values.
In "latent" mode, latent low dimensional representation of cells are stored
in `adata.obsm['X_dca']` and `adata.X` is not modified.
Note that these values are not corrected for library size effects.
If `return_info` is true, all estimated distribution parameters are stored
in AnnData like this:
`.obsm["X_dca_dropout"]`
The mixture coefficient (pi) of the zero component in ZINB,
i.e. dropout probability (if `ae_type` is `zinb` or `zinb-conddisp`).
`.obsm["X_dca_dispersion"]`
The dispersion parameter of NB.
`.uns["dca_loss_history"]`
The loss history of the training.
See `.history` attribute of Keras History class for mode details.
Finally, the raw counts are stored in `.raw` attribute of AnnData object.
If `return_model` is given, trained model is returned.
When both `copy` and `return_model` are true,
a tuple of anndata and model is returned in that order.
"""
try:
from dca.api import dca
except ImportError:
raise ImportError(
'Please install dca package (>= 0.2.1) via `pip install dca`')
return dca(
adata,
mode=mode,
ae_type=ae_type,
normalize_per_cell=normalize_per_cell,
scale=scale,
log1p=log1p,
hidden_size=hidden_size,
hidden_dropout=hidden_dropout,
batchnorm=batchnorm,
activation=activation,
init=init,
network_kwds=network_kwds,
epochs=epochs,
reduce_lr=reduce_lr,
early_stop=early_stop,
batch_size=batch_size,
optimizer=optimizer,
random_state=random_state,
threads=threads,
learning_rate=learning_rate,
verbose=verbose,
training_kwds=training_kwds,
return_model=return_model,
return_info=return_info,
copy=copy,
)
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--n_clusters', default=3, type=int)
parser.add_argument('--data_file', default=None)
args = parser.parse_args()
data_mat = h5py.File('./normalized_raw_data/' + str(args.data_file))
x = np.array(data_mat['X'])
y = np.array(data_mat['Y'])
adata = sc.AnnData(x)
adata.obs['Group'] = y
adata = read_dataset(adata, transpose=False, test_split=False, copy=True)
sc.pp.filter_genes(adata, min_counts=1)
dca(adata, threads=1)
sc.pp.normalize_per_cell(adata)
sc.pp.log1p(adata)
sc.pp.pca(adata)
print(adata)
dca_pca = adata.obsm.X_pca[:, :2]
kmeans = KMeans(n_clusters=args.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(dca_pca)
acc = np.round(cluster_acc(y, y_pred), 5)
nmi = np.round(metrics.normalized_mutual_info_score(y, y_pred), 5)
ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5)
print('data: ' + str(args.data_file) +
' DCA+PCA+kmeans: ACC= %.4f, NMI= %.4f, ARI= %.4f' % (acc, nmi, ari))
os.makedirs(output_dir, exist_ok=True)
starttime = time.time()
gene_bc_mat, cell_id, gene_name = read_loom(FLAGS["loom"])
min_expressed_cell = FLAGS["min_expressed_cell"]
min_expressed_cell_average_expression = FLAGS[
"min_expressed_cell_average_expression"]
expressed_cell = (gene_bc_mat > 0).sum(1)
gene_expression = gene_bc_mat.sum(1)
gene_filter = np.bitwise_and(
expressed_cell >= min_expressed_cell,
gene_expression > expressed_cell * min_expressed_cell_average_expression)
input_gene_bc_mat = gene_bc_mat[gene_filter, :]
print(input_gene_bc_mat.shape)
filt_adata = sc.AnnData(input_gene_bc_mat.transpose())
dca(filt_adata)
input_loom_name = FLAGS["loom"].rsplit("/", 1)[1]
output_h5 = input_loom_name.replace(
".loom",
"_DCA_mc_{}_mce_{}.hdf5".format(min_expressed_cell,
min_expressed_cell_average_expression))
with h5py.File("{}/{}".format(output_dir, output_h5), "w") as f:
f["cell_id"] = cell_id.astype(h5py.special_dtype(vlen=str))
f["gene_name"] = gene_name[gene_filter].astype(
h5py.special_dtype(vlen=str))
if_dset_imputation = f.create_dataset("imputation",
shape=(cell_id.size,
gene_filter.sum()),
chunks=(1, gene_filter.sum()),
dtype=np.float32)
if_dset_imputation[...] = filt_adata.X
