def make_dataloader(rpkm, tnf, batchsize=256, destroy=False, cuda=True):
mask = tnf.sum(axis=1) != 0
if rpkm.shape[1] > 1:
depthssum = rpkm.sum(axis=1)
mask &= depthssum != 0
depthssum = depthssum[mask]
if destroy:
rpkm = numpy_inplace_maskarray(rpkm, mask)
tnf = numpy_inplace_maskarray(tnf, mask)
else:
rpkm = rpkm[mask].astype(_np.float32, copy=False)
tnf = tnf[mask].astype(_np.float32, copy=False)
if rpkm.shape[1] > 1:
rpkm /= depthssum.reshape((-1, 1))
else:
zscore(rpkm, axis=0, inplace=True)
zscore(tnf, axis=0, inplace=True)
depthstensor = _torch.from_numpy(rpkm)
tnftensor = _torch.from_numpy(tnf)
# 创建dataloader
n_workers = 4 if cuda else 1
dataset = _TensorDataset(depthstensor, tnftensor)
dataloader = _DataLoader(dataset=dataset, batch_size=batchsize, drop_last=True,
shuffle=True, num_workers=n_workers, pin_memory=cuda)
return dataloader
def make_dataloader(rpkm, tnf, batchsize=256, destroy=False, cuda=False):
"""Create a DataLoader and a contig mask from RPKM and TNF.
The dataloader is an object feeding minibatches of contigs to the VAE.
The data are normalized versions of the input datasets, with zero-contigs,
i.e. contigs where a row in either TNF or RPKM are all zeros, removed.
The mask is a boolean mask designating which contigs have been kept.
Inputs:
rpkm: RPKM matrix (N_contigs x N_samples)
tnf: TNF matrix (N_contigs x N_TNF)
batchsize: Starting size of minibatches for dataloader
destroy: Mutate rpkm and tnf array in-place instead of making a copy.
cuda: Pagelock memory of dataloader (use when using GPU acceleration)
Outputs:
DataLoader: An object feeding data to the VAE
mask: A boolean mask of which contigs are kept
"""
if not isinstance(rpkm, _np.ndarray) or not isinstance(tnf, _np.ndarray):
raise ValueError('TNF and RPKM must be Numpy arrays')
if batchsize < 1:
raise ValueError('Minimum batchsize of 1, not {}'.format(batchsize))
if len(rpkm) != len(tnf):
raise ValueError('Lengths of RPKM and TNF must be the same')
if not (rpkm.dtype == tnf.dtype == _np.float32):
raise ValueError('TNF and RPKM must be Numpy arrays of dtype float32')
mask = tnf.sum(axis=1) != 0
# If multiple samples, also include nonzero depth as requirement for accept
# of sequences
if rpkm.shape[1] > 1:
depthssum = rpkm.sum(axis=1)
mask &= depthssum != 0
depthssum = depthssum[mask]
if mask.sum() < batchsize:
raise ValueError(
'Fewer sequences left after filtering than the batch size.')
if destroy:
rpkm = _vambtools.numpy_inplace_maskarray(rpkm, mask)
tnf = _vambtools.numpy_inplace_maskarray(tnf, mask)
else:
# The astype operation does not copy due to "copy=False", but the masking
# operation does.
rpkm = rpkm[mask].astype(_np.float32, copy=False)
tnf = tnf[mask].astype(_np.float32, copy=False)
# If multiple samples, normalize to sum to 1, else zscore normalize
if rpkm.shape[1] > 1:
rpkm /= depthssum.reshape((-1, 1))
else:
_vambtools.zscore(rpkm, axis=0, inplace=True)
# Normalize arrays and create the Tensors (the tensors share the underlying memory)
# of the Numpy arrays
_vambtools.zscore(tnf, axis=0, inplace=True)
depthstensor = _torch.from_numpy(rpkm)
tnftensor = _torch.from_numpy(tnf)
# Create dataloader
n_workers = 4 if cuda else 1
dataset = _TensorDataset(depthstensor, tnftensor)
dataloader = _DataLoader(dataset=dataset,
batch_size=batchsize,
drop_last=True,
shuffle=True,
num_workers=n_workers,
pin_memory=cuda)
return dataloader, mask
def make_dataloader(rpkm, tnf, batchsize=64, destroy=False, cuda=False):
"""Create a DataLoader and a contig mask from RPKM and TNF.
The dataloader is an object feeding minibatches of contigs to the VAE.
The data are normalized versions of the input datasets, with zero-contigs,
i.e. contigs where a row in either TNF or RPKM are all zeros, removed.
The mask is a boolean mask designating which contigs have been kept.
Inputs:
rpkm: RPKM matrix (N_contigs x N_samples)
tnf: TNF matrix (N_contigs x 136)
batchsize: Starting size of minibatches for dataloader
destroy: Mutate rpkm and tnf array in-place instead of making a copy.
cuda: Pagelock memory of dataloader (use when using GPU acceleration)
Outputs:
DataLoader: An object feeding data to the VAE
mask: A boolean mask of which contigs are kept
"""
if not isinstance(rpkm, _np.ndarray) or not isinstance(tnf, _np.ndarray):
raise ValueError('TNF and RPKM must be Numpy arrays')
if batchsize < 1:
raise ValueError('Minimum batchsize of 1, not {}'.format(batchsize))
if len(rpkm) != len(tnf):
raise ValueError('Lengths of RPKM and TNF must be the same')
if tnf.shape[1] != 136:
raise ValueError('TNF must be 136 long along axis 1')
tnfsum = tnf.sum(axis=1)
mask = tnfsum != 0
del tnfsum
depthssum = rpkm.sum(axis=1)
mask &= depthssum != 0
if destroy:
if not (rpkm.dtype == tnf.dtype == _np.float32):
raise ValueError(
'Arrays must be of data type np.float32 if destroy is True')
rpkm = _vambtools.inplace_maskarray(rpkm, mask)
tnf = _vambtools.inplace_maskarray(tnf, mask)
else:
rpkm = rpkm[mask].astype(_np.float32, copy=False)
tnf = tnf[mask].astype(_np.float32, copy=False)
depthssum = depthssum[mask]
# Normalize arrays and create the Tensors
rpkm /= depthssum.reshape((-1, 1))
_vambtools.zscore(tnf, axis=0, inplace=True)
depthstensor = _torch.from_numpy(rpkm)
tnftensor = _torch.from_numpy(tnf)
# Create dataloader
dataset = _TensorDataset(depthstensor, tnftensor)
dataloader = _DataLoader(dataset=dataset,
batch_size=batchsize,
drop_last=True,
shuffle=True,
num_workers=1,
pin_memory=cuda)
return dataloader, mask