def __setitem__(self, index, value):
interval = index[0]
condition = index[1]
if isinstance(interval, GenomicInterval) and isinstance(condition, int):
chrom = interval.chrom
start = interval.start // self.resolution
end = int(numpy.ceil(interval.end / self.resolution))
strand = interval.strand
sind = 1 if self.stranded and strand == '-' else 0
for idx, iarray in enumerate(range(start, end)):
if hasattr(value, '__len__'):
# value is a numpy array or a list
val = value[idx]
else:
# value is a scalar value
val = value
if val > 0:
if not self._full_genome_stored:
self.handle[_iv_to_str(chrom, interval.start,
interval.end)][idx,
sind * len(self.condition)
+ condition] = val
else:
self.handle[chrom][iarray,
sind * len(self.condition)
+ condition] = val
return
raise IndexError("Index must be a GenomicInterval and a condition index")
def _setitem(self, interval, condition, length, value):
if not self._full_genome_stored:
regidx = self.region2index[_iv_to_str(interval.chrom,
interval.start,
interval.end)]
nconditions = len(self.condition)
ncondstrand = len(self.condition) * value.shape[-1]
#end = end - self.order + 1
idxs = np.where(value > 0)
for idx in zip(*idxs):
basepos = idx[0] * ncondstrand
strand = idx[1] * nconditions
cond = condition if isinstance(condition, int) else idx[2]
self.handle['data'][regidx,
basepos + strand + cond] = value[idx]
else:
ref_start, ref_end, array_start, _ = self._get_indices(
interval, value.shape[0])
idxs = np.where(value > 0)
iarray = np.arange(ref_start, ref_end)
for idx in zip(*idxs):
cond = condition if isinstance(condition, int) else idx[2]
self.handle[interval.chrom][iarray[idx[0]],
idx[1] * len(self.condition) +
cond] = value[idx[0] +
array_start][idx[1:]]
def __getitem__(self, index):
# for now lets ignore everything except for chrom, start and end.
if isinstance(index, Interval):
interval = index
chrom = interval.chrom
start = self.get_iv_start(interval.start)
end = self.get_iv_end(interval.end)
# original length
length = end - start - self.order + 1
if not self._full_genome_stored:
idx = self.region2index[_iv_to_str(chrom, interval.start,
interval.end)]
# correcting for the overshooting starts and ends is not necessary
# for partially loaded data
return self._reshape(
self.handle['data'][idx],
(length, 2 if self.stranded else 1, len(self.condition)))
if chrom not in self.handle:
return np.ones(
(length, 2 if self.stranded else 1, len(self.condition)),
dtype=self.typecode) * self.padding_value
if start >= 0 and end <= self.handle[chrom].shape[0]:
end = end - self.order + 1
# this is a short-cut, which does not require zero-padding
return self._reshape(self.handle[chrom][start:end],
(end - start, 2 if self.stranded else 1,
len(self.condition)))
# below is some functionality for zero-padding, in case the region
# reaches out of the chromosome size
if self.padding_value == 0.0:
data = np.zeros(
(length, 2 if self.stranded else 1, len(self.condition)),
dtype=self.typecode)
else:
data = np.ones(
(length, 2 if self.stranded else 1, len(self.condition)),
dtype=self.typecode) * self.padding_value
ref_start, ref_end, array_start, array_end = self._get_indices(
interval, data.shape[0])
data[array_start:array_end, :, :] = self._reshape(
self.handle[chrom][ref_start:ref_end],
(ref_end - ref_start, 2 if self.stranded else 1,
len(self.condition)))
return data
raise IndexError("Cannot interpret interval: {}".format(index))
def __getitem__(self, index):
# for now lets ignore everything except for chrom, start and end.
if isinstance(index, GenomicInterval):
interval = index
chrom = interval.chrom
start = self.get_iv_start(interval.start)
end = self.get_iv_end(interval.end)
# original length
length = end - start
if not self._full_genome_stored:
# correcting for the overshooting starts and ends is not necessary
# for partially loaded data
return self._reshape(
self.handle[_iv_to_str(chrom, interval.start,
interval.end)][:(length)],
(length, 2 if self.stranded else 1, len(self.condition)))
if start >= 0 and end <= self.handle[chrom].shape[0]:
# this is a short-cut, which does not require zero-padding
return self._reshape(self.handle[chrom][start:end],
(end - start, 2 if self.stranded else 1,
len(self.condition)))
# below is some functionality for zero-padding, in case the region
# reaches out of the chromosome size
data = np.zeros(
(length, 2 if self.stranded else 1, len(self.condition)),
dtype=self.handle[chrom].dtype)
dstart = 0
dend = length
# if start of interval is negative, due to flank, discard the start
if start < 0:
dstart = -start
start = 0
# if end of interval reached out of the chromosome, clip it
if self.handle[chrom].shape[0] < end:
dend -= end - self.handle[chrom].shape[0]
end = self.handle[chrom].shape[0]
# dstart and dend are offset by the number of positions
# the region reaches out of the chromosome
data[dstart:dend, :, :] = self._reshape(
self.handle[chrom][start:end],
(end - start, 2 if self.stranded else 1, len(self.condition)))
return data
raise IndexError("Index must be a GenomicInterval")
def _setitem(self, interval, condition, length, value):
if not self._full_genome_stored:
idx = self.region2index[_iv_to_str(interval.chrom, interval.start,
interval.end)]
# correcting for the overshooting starts and ends is not necessary
# for partially loaded data
self.handle['data'][idx, :length, :, condition] = value
else:
ref_start, ref_end, array_start, \
array_end = self._get_indices(interval, value.shape[0])
self.handle[interval.chrom][ref_start:ref_end, :, condition] = \
value[array_start:array_end]
def __setitem__(self, index, value):
interval = index[0]
condition = index[1]
if isinstance(interval, GenomicInterval) and isinstance(condition, int):
chrom = interval.chrom
start = interval.start // self.resolution
end = int(numpy.ceil(interval.end / self.resolution))
strand = interval.strand
try:
if not self._full_genome_stored:
length = end-start
# correcting for the overshooting starts and ends is not necessary
# for partially loaded data
self.handle[_iv_to_str(chrom, interval.start,
interval.end)][:(length),
1 if self.stranded and strand == '-' else 0,
condition] = value
# raise IndexError('Region {} not '.format(_iv_to_str(
# chrom, interval.start, interval.end)) +
# 'contained in the genomic array. '
# 'Consider adjusting the regions, '
# 'binsize, stepsize and flank.')
else:
self.handle[chrom][start:end,
1 if self.stranded and strand == '-' else 0,
condition] = value
except KeyError:
print('Skipping region {} - not in genomic array.'.format(
_iv_to_str(chrom, interval.start, interval.end)) +
'Consider using store_whole_genome=True or '
'adjusting adjusting the regions, binsize, stepsize and flank.')
else:
raise IndexError("Index must be a GenomicInterval and a condition index")
def load_sequence(self):
print('loading from lazy loader')
store_whole_genome = self.store_whole_genome
gindexer = self.gindexer
if isinstance(self.fastafile, str):
seqs = sequences_from_fasta(self.fastafile, self.seqtype)
else:
# This is already a list of SeqRecords
seqs = self.fastafile
if not store_whole_genome and gindexer is not None:
# the genome is loaded with a bed file,
# only the specific subset is loaded
# to keep the memory overhead low.
# Otherwise the entire reference genome is loaded.
rgen = OrderedDict(((seq.id, seq) for seq in seqs))
subseqs = []
for giv in gindexer:
subseq = rgen[giv.chrom][
max(giv.start, 0):min(giv.end, len(rgen[giv.chrom]))]
if giv.start < 0:
subseq = 'N' * (-giv.start) + subseq
if len(subseq) < giv.length:
subseq = subseq + 'N' * (giv.length - len(subseq))
subseq.id = _iv_to_str(giv.chrom, giv.start, giv.end)
subseq.name = subseq.id
subseq.description = subseq.id
subseqs.append(subseq)
seqs = subseqs
gsize = gindexer
if store_whole_genome:
gsize = OrderedDict(((seq.id, len(seq)) for seq in seqs))
gsize = GenomicIndexer.create_from_genomesize(gsize)
self.gsize_ = gsize
self.seqs_ = seqs
def __init__(
self,
gsize, # pylint: disable=too-many-locals
stranded=True,
conditions=None,
typecode='d',
datatags=None,
resolution=1,
order=1,
padding_value=0.0,
store_whole_genome=True,
cache=None,
overwrite=False,
loader=None,
normalizer=None,
collapser=None):
super(NPGenomicArray,
self).__init__(stranded,
conditions,
typecode,
resolution,
order=order,
padding_value=padding_value,
store_whole_genome=store_whole_genome,
collapser=collapser)
gsize_ = None
if not store_whole_genome:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
self.region2index = {_iv_to_str(region.chrom,
region.start,
region.end): i \
for i, region in enumerate(gsize_)}
cachefile = _get_cachefile(cache, datatags, '.npz')
load_from_file = _load_data(cache, datatags, '.npz')
if load_from_file:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
if store_whole_genome:
data = {
str(region.chrom): init_with_padding_value(
padding_value,
shape=(_get_iv_length(region.length - self.order + 1,
self.resolution),
2 if stranded else 1, len(self.condition)),
dtype=self.typecode)
for region in gsize_
}
names = [str(region.chrom) for region in gsize_]
self.handle = data
else:
data = {
'data':
init_with_padding_value(
padding_value,
shape=(len(gsize_),
_get_iv_length(
gsize_.binsize + 2 * gsize_.flank -
self.order + 1, self.resolution)
if self.resolution is not None else 1,
2 if stranded else 1, len(self.condition)),
dtype=self.typecode)
}
names = ['data']
self.handle = data
# invoke the loader
if loader:
loader(self)
if cachefile is not None:
np.savez(cachefile, **data)
if cachefile is not None:
print('reload {}'.format(cachefile))
data = np.load(cachefile)
names = [x for x in data]
# here we get either the freshly loaded data or the reloaded
# data from np.load.
self.handle = {key: data[key] for key in names}
for norm in normalizer or []:
get_normalizer(norm)(self)
def __init__(
self,
gsize, # pylint: disable=too-many-locals
stranded=True,
conditions=None,
typecode='d',
datatags=None,
resolution=1,
order=1,
padding_value=0.,
store_whole_genome=True,
cache=None,
overwrite=False,
loader=None,
normalizer=None,
collapser=None):
super(HDF5GenomicArray,
self).__init__(stranded,
conditions,
typecode,
resolution,
order=order,
padding_value=padding_value,
store_whole_genome=store_whole_genome,
collapser=collapser)
if cache is None:
raise ValueError('HDF5 format requires cache=True')
gsize_ = None
if not store_whole_genome:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
self.region2index = {_iv_to_str(region.chrom,
region.start,
region.end): i \
for i, region in enumerate(gsize_)}
cachefile = _get_cachefile(cache, datatags, '.h5')
load_from_file = _load_data(cache, datatags, '.h5')
if load_from_file:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
h5file = h5py.File(cachefile, 'w')
if store_whole_genome:
for region in gsize_:
shape = (_get_iv_length(region.length - self.order + 1,
self.resolution),
2 if stranded else 1, len(self.condition))
h5file.create_dataset(str(region.chrom),
shape,
dtype=self.typecode,
data=init_with_padding_value(
padding_value, shape,
self.typecode))
self.handle = h5file
else:
shape = (len(gsize_),
_get_iv_length(
gsize_.binsize + 2 * gsize_.flank - self.order +
1, self.resolution), 2 if stranded else 1,
len(self.condition))
h5file.create_dataset('data',
shape,
dtype=self.typecode,
data=init_with_padding_value(
padding_value, shape, self.typecode))
self.handle = h5file
# invoke the loader
if loader:
loader(self)
for norm in normalizer or []:
get_normalizer(norm)(self)
h5file.close()
print('reload {}'.format(cachefile))
h5file = h5py.File(cachefile, 'a', driver='stdio')
self.handle = h5file
def __setitem__(self, index, value):
interval = index[0]
condition = index[1]
if self.stranded and value.shape[-1] != 2:
raise ValueError(
'If genomic array is in stranded mode, shape[-1] == 2 is expected'
)
if not self.stranded and value.shape[-1] != 1:
value = value.sum(axis=1).reshape(-1, 1)
if isinstance(interval, GenomicInterval) and isinstance(
condition, int):
chrom = interval.chrom
start = self.get_iv_start(interval.start)
end = self.get_iv_end(interval.end)
# value should be a 2 dimensional array
# it will be reshaped to a 2D array where the collapse operation is performed
# along the second dimension.
if self.collapser is not None:
if self.resolution is None:
# collapse along the entire interval
value = value.reshape((1, len(value), value.shape[-1]))
else:
# collapse in bins of size resolution
value = value.reshape((len(value) // self.resolution,
self.resolution, value.shape[-1]))
value = self.collapser(value)
try:
if not self._full_genome_stored:
length = end - start
# correcting for the overshooting starts and ends is not necessary
# for partially loaded data
self.handle[_iv_to_str(chrom, interval.start,
interval.end)][:(length), :,
condition] = value
else:
if start < 0:
tmp_start = -start
ref_start = 0
else:
tmp_start = 0
ref_start = start
if end > self.handle[chrom].shape[0]:
tmp_end = value.shape[0] - (
end - self.handle[chrom].shape[0])
ref_end = self.handle[chrom].shape[0]
else:
tmp_end = value.shape[0]
ref_end = end
#start_offset = max(start, 0)
#end_offset = min(end, self.handle[chrom].shape[0])
#dstart = start_offset - start
#dend = end_offset - end
#cend = end + (dend)
#if dend < 0:
self.handle[chrom][ref_start:ref_end, :, condition] = \
value[tmp_start:tmp_end, :]
except KeyError:
# we end up here if the peak regions are not a subset of
# the regions of interest. that might be the case if
# peaks from the holdout proportion of the genome are tried
# to be added.
# unfortunately, it is also possible that store_whole_genome=False
# and the peaks and regions of interest are just not synchronized
# in which case nothing (or too few peaks) are added. in the latter
# case an error would help actually, but I am not sure how to
# check if the first or the second is the case here.
pass
else:
raise IndexError(
"Index must be a GenomicInterval and a condition index")
def __setitem__(self, index, value):
interval = index[0]
condition = index[1]
if isinstance(interval, GenomicInterval) and isinstance(
condition, int):
chrom = interval.chrom
start = self.get_iv_start(interval.start)
end = self.get_iv_end(interval.end)
#strand = interval.strand
#sind = 1 if self.stranded and strand == '-' else 0
if self.stranded and value.shape[-1] != 2:
raise ValueError(
'If genomic array is in stranded mode, shape[-1] == 2 is expected'
)
if not self.stranded and value.shape[-1] != 1:
value = value.sum(axis=1).reshape(-1, 1)
# value should be a 2 dimensional array
# it will be reshaped to a 2D array where the collapse operation is performed
# along the second dimension.
if self.collapser is not None:
if self.resolution is None:
# collapse along the entire interval
value = value.reshape((1, len(value), value.shape[-1]))
else:
# collapse in bins of size resolution
value = value.reshape((len(value) // self.resolution,
self.resolution, value.shape[-1]))
value = self.collapser(value)
try:
for sind in range(value.shape[-1]):
if not self._full_genome_stored:
for idx, iarray in enumerate(range(start, end)):
val = value[idx, sind]
if val > 0:
self.handle[_iv_to_str(
chrom, interval.start,
interval.end)][idx,
sind * len(self.condition) +
condition] = val
else:
if start < 0:
tmp_start = -start
ref_start = 0
else:
tmp_start = 0
ref_start = start
if end > self.handle[chrom].shape[0]:
tmp_end = value.shape[0] - (
end - self.handle[chrom].shape[0])
ref_end = self.handle[chrom].shape[0]
else:
tmp_end = value.shape[0]
ref_end = end
for idx, iarray in enumerate(range(ref_start,
ref_end)):
val = value[idx + tmp_start, sind]
if val > 0:
self.handle[chrom][iarray,
sind * len(self.condition) +
condition] = val
except KeyError:
# we end up here if the peak regions are not a subset of
# the regions of interest. that might be the case if
# peaks from the holdout proportion of the genome are tried
# to be added.
# unfortunately, it is also possible that store_whole_genome=False
# and the peaks and regions of interest are just not synchronized
# in which case nothing (or too few peaks) are added. in the latter
# case an error would help actually, but I am not sure how to
# check if the first or the second is the case here.
pass
return
raise IndexError(
"Index must be a GenomicInterval and a condition index")
def create_from_bed(
cls,
name, # pylint: disable=too-many-locals
bedfiles,
regions=None,
genomesize=None,
conditions=None,
binsize=None,
stepsize=None,
resolution=1,
flank=0,
storage='ndarray',
dtype='int',
dimmode='all',
mode='binary',
store_whole_genome=False,
overwrite=False,
channel_last=True,
datatags=None,
cache=False):
"""Create a Cover class from a bed-file (or files).
Parameters
-----------
name : str
Name of the dataset
bedfiles : str or list
bed-file or list of bed files.
regions : str or None
Bed-file defining the region of interest.
If set to None a genomesize must be supplied and
a genomic indexer must be attached later.
genomesize : dict or None
Dictionary containing the genome size to fetch the coverage from.
If `genomesize=None`, the genome size
is fetched from the region of interest.
conditions : list(str) or None
List of conditions.
If `conditions=None`,
the conditions are obtained from
the filenames (without the directories
and file-ending).
binsize : int or None
Binsize in basepairs. For binsize=None,
the binsize will be determined from the bed-file directly
which requires that all intervals in the bed-file are of equal
length. Otherwise, the intervals in the bed-file will be
split to subintervals of length binsize in conjunction with
stepsize. Default: None.
stepsize : int or None
stepsize in basepairs for traversing the genome.
If stepsize is None, it will be set equal to binsize.
Default: None.
resolution : int
Resolution in base pairs divides the region of interest
in windows of length resolution.
This effectively reduces the storage for coverage data.
The resolution must be selected such that min(stepsize, binsize)
is a multiple of resolution.
Default: 1.
flank : int
Flanking size increases the interval size at both ends by
flank bins. Note that the binsize is defined by the resolution parameter.
Default: 0.
storage : str
Storage mode for storing the coverage data can be
'ndarray', 'hdf5' or 'sparse'. Default: 'ndarray'.
dtype : str
Typecode to define the datatype to be used for storage.
Default: 'int'.
dimmode : str
Dimension mode can be 'first' or 'all'. If 'first', only
the first element of size resolution is returned. Otherwise,
all elements of size resolution spanning the interval are returned.
Default: 'all'.
mode : str
Mode of the dataset may be 'binary', 'score' or 'categorical'.
Default: 'binary'.
overwrite : boolean
Overwrite cachefiles. Default: False.
datatags : list(str) or None
List of datatags. Together with the dataset name,
the datatags are used to construct a cache file.
If :code:`cache=False`, this option does not have an effect.
Default: None.
store_whole_genome : boolean
Indicates whether the whole genome or only selected regions
should be loaded. If False, a bed-file with regions of interest
must be specified. Default: False.
channel_last : boolean
Indicates whether the condition axis should be the last dimension
or the first. For example, tensorflow expects the channel at the
last position. Default: True.
cache : boolean
Indicates whether to cache the dataset. Default: False.
"""
if regions is None and genomesize is None:
raise ValueError('Either regions or genomesize must be specified.')
if regions is not None:
gindexer = GenomicIndexer.create_from_file(regions, binsize,
stepsize, flank)
else:
gindexer = None
if not store_whole_genome:
# if whole genome should not be loaded
gsize = {
_iv_to_str(iv.chrom, iv.start, iv.end): iv.end - iv.start
for iv in gindexer
}
else:
# otherwise the whole genome will be fetched, or at least
# a set of full length chromosomes
if genomesize is not None:
# if a genome size has specifically been given, use it.
gsize = genomesize.copy()
else:
gsize = get_genome_size_from_regions(regions)
if isinstance(bedfiles, str):
bedfiles = [bedfiles]
if mode == 'categorical':
if len(bedfiles) > 1:
raise ValueError('Only one bed-file is '
'allowed with mode=categorical')
sample_file = bedfiles[0]
regions_ = _get_genomic_reader(sample_file)
max_class = 0
for reg in regions_:
if reg.score > max_class:
max_class = reg.score
if conditions is None:
conditions = [str(i) for i in range(int(max_class + 1))]
if conditions is None:
conditions = [
os.path.splitext(os.path.basename(f))[0] for f in bedfiles
]
def _bed_loader(garray, bedfiles, genomesize, mode):
print("load from bed")
for i, sample_file in enumerate(bedfiles):
regions_ = _get_genomic_reader(sample_file)
for region in regions_:
gidx = GenomicIndexer.create_from_region(
region.iv.chrom, region.iv.start, region.iv.end,
region.iv.strand, binsize, stepsize, flank)
for greg in gidx:
if region.score is None and mode in [
'score', 'categorical'
]:
raise ValueError(
'No Score available. Score field must '
'present in {}'.format(sample_file) + \
'for mode="{}"'.format(mode))
# if region score is not defined, take the mere
# presence of a range as positive label.
if mode == 'score':
garray[greg,
i] = np.dtype(dtype).type(region.score)
elif mode == 'categorical':
garray[greg,
int(region.score)] = np.dtype(dtype).type(1)
elif mode == 'binary':
garray[greg, i] = np.dtype(dtype).type(1)
return garray
# At the moment, we treat the information contained
# in each bed-file as unstranded
datatags = [name] + datatags if datatags else [name]
datatags += ['resolution{}'.format(resolution)]
cover = create_genomic_array(gsize,
stranded=False,
storage=storage,
datatags=datatags,
cache=cache,
conditions=conditions,
resolution=resolution,
overwrite=overwrite,
typecode=dtype,
store_whole_genome=store_whole_genome,
loader=_bed_loader,
loader_args=(bedfiles, gsize, mode))
return cls(name,
cover,
gindexer,
padding_value=0,
dimmode=dimmode,
channel_last=channel_last)
def create_from_bam(
cls,
name, # pylint: disable=too-many-locals
bamfiles,
regions=None,
genomesize=None,
conditions=None,
min_mapq=None,
binsize=None,
stepsize=None,
flank=0,
resolution=1,
storage='ndarray',
dtype='int',
stranded=True,
overwrite=False,
pairedend='5prime',
template_extension=0,
aggregate=None,
datatags=None,
cache=False,
channel_last=True,
store_whole_genome=False):
"""Create a Cover class from a bam-file (or files).
This constructor can be used to obtain coverage from BAM files.
For single-end reads the read will be counted at the 5 prime end.
Paired-end reads can be counted relative to the 5 prime ends of the read
(default) or with respect to the midpoint.
Parameters
-----------
name : str
Name of the dataset
bamfiles : str or list
bam-file or list of bam files.
regions : str or None
Bed-file defining the region of interest.
If set to None, the coverage will be
fetched from the entire genome and a
genomic indexer must be attached later.
genomesize : dict or None
Dictionary containing the genome size.
If `genomesize=None`, the genome size
is determined from the bam header.
If `store_whole_genome=False`, this option does not have an effect.
conditions : list(str) or None
List of conditions.
If `conditions=None`,
the conditions are obtained from
the filenames (without the directories
and file-ending).
min_mapq : int
Minimal mapping quality.
Reads with lower mapping quality are
filtered out. If None, all reads are used.
binsize : int or None
Binsize in basepairs. For binsize=None,
the binsize will be determined from the bed-file directly
which requires that all intervals in the bed-file are of equal
length. Otherwise, the intervals in the bed-file will be
split to subintervals of length binsize in conjunction with
stepsize. Default: None.
stepsize : int or None
stepsize in basepairs for traversing the genome.
If stepsize is None, it will be set equal to binsize.
Default: None.
flank : int
Flanking size increases the interval size at both ends by
flank base pairs. Default: 0
resolution : int
Resolution in base pairs divides the region of interest
in windows of length resolution.
This effectively reduces the storage for coverage data.
The resolution must be selected such that min(stepsize, binsize)
is a multiple of resolution.
Default: 1.
storage : str
Storage mode for storing the coverage data can be
'ndarray', 'hdf5' or 'sparse'. Default: 'ndarray'.
dtype : str
Typecode to be used for storage the data.
Default: 'int'.
stranded : boolean
Indicates whether to extract stranded or
unstranded coverage. For unstranded
coverage, reads aligning to both strands will be aggregated.
overwrite : boolean
Overwrite cachefiles. Default: False.
datatags : list(str) or None
List of datatags. Together with the dataset name,
the datatags are used to construct a cache file.
If :code:`cache=False`, this option does not have an effect.
Default: None.
pairedend : str
Indicates whether to count reads at the '5prime' end or at
the 'midpoint' for paired-end reads. Default: '5prime'.
template_extension : int
Elongates intervals by template_extension which allows to properly count
template mid-points whose reads lie outside of the interval.
This option is only relevant for paired-end reads counted at the
'midpoint' and if the coverage is not obtained from the
whole genome, e.g. regions is not None.
aggregate : callable or None
Aggregation operation for loading genomic array. If None,
the coverage amounts to the raw counts.
Default: None
cache : boolean
Indicates whether to cache the dataset. Default: False.
channel_last : boolean
Indicates whether the condition axis should be the last dimension
or the first. For example, tensorflow expects the channel at the
last position. Default: True.
store_whole_genome : boolean
Indicates whether the whole genome or only selected regions
should be loaded. If False, a bed-file with regions of interest
must be specified. Default: False
"""
if pysam is None: # pragma: no cover
raise Exception(
'pysam not available. '
'`create_from_bam` requires pysam to be installed.')
if regions is not None:
gindexer = GenomicIndexer.create_from_file(regions, binsize,
stepsize, flank)
else:
gindexer = None
if isinstance(bamfiles, str):
bamfiles = [bamfiles]
if conditions is None:
conditions = [
os.path.splitext(os.path.basename(f))[0] for f in bamfiles
]
if min_mapq is None:
min_mapq = 0
full_genome_index = store_whole_genome
if not full_genome_index and not gindexer:
raise ValueError(
'Either regions must be supplied or store_whole_genome must be True'
)
if not full_genome_index:
# if whole genome should not be loaded
gsize = {
_iv_to_str(iv.chrom, iv.start, iv.end): iv.end - iv.start
for iv in gindexer
}
else:
# otherwise the whole genome will be fetched, or at least
# a set of full length chromosomes
if genomesize is not None:
# if a genome size has specifically been given, use it.
gsize = genomesize.copy()
else:
header = pysam.AlignmentFile(bamfiles[0], 'r') # pylint: disable=no-member
gsize = {}
for chrom, length in zip(header.references, header.lengths):
gsize[chrom] = length
def _bam_loader(garray, files):
print("load from bam")
for i, sample_file in enumerate(files):
print('Counting from {}'.format(sample_file))
aln_file = pysam.AlignmentFile(sample_file, 'rb') # pylint: disable=no-member
for chrom in gsize:
array = np.zeros(
(get_chrom_length(gsize[chrom], resolution), 2),
dtype=dtype)
locus = _str_to_iv(chrom,
template_extension=template_extension)
if len(locus) == 1:
locus = (locus[0], 0, gsize[chrom])
# locus = (chr, start, end)
# or locus = (chr, )
for aln in aln_file.fetch(*locus):
if aln.is_unmapped:
continue
if aln.mapq < min_mapq:
continue
if aln.is_read2:
# only consider read1 so as not to double count
# fragments for paired end reads
# read2 will also be false for single end
# reads.
continue
if aln.is_paired:
# if paired end read, consider the midpoint
if not (aln.is_proper_pair and aln.reference_name
== aln.next_reference_name):
# only consider paired end reads if both mates
# are properly mapped and they map to the
# same reference_name
continue
# if the next reference start >= 0,
# the read is considered as a paired end read
# in this case we consider the mid point
if pairedend == 'midpoint':
pos = min(aln.reference_start,
aln.next_reference_start) + \
abs(aln.template_length) // 2
else:
if aln.is_reverse:
# last position of the downstream read
pos = max(
aln.reference_end,
aln.next_reference_start +
aln.query_length)
else:
# first position of the upstream read
pos = min(aln.reference_start,
aln.next_reference_start)
else:
# here we consider single end reads
# whose 5 prime end is determined strand specifically
if aln.is_reverse:
pos = aln.reference_end
else:
pos = aln.reference_start
if not garray._full_genome_stored:
# if we get here, a region was given,
# otherwise, the entire chromosome is read.
pos -= locus[1] + template_extension
if pos < 0 or pos >= locus[2] - locus[1]:
# if the read 5 p end or mid point is outside
# of the region of interest, the read is discarded
continue
# compute divide by the resolution
pos //= resolution
# fill up the read strand specifically
if aln.is_reverse:
array[pos, 1] += 1
else:
array[pos, 0] += 1
# apply the aggregation
if aggregate is not None:
array = aggregate(array)
if stranded:
lp = locus + ('+', )
garray[GenomicInterval(*lp), i] = array[:, 0]
lm = locus + ('-', )
garray[GenomicInterval(*lm), i] = array[:, 1]
else:
# if unstranded, aggregate the reads from
# both strands
garray[GenomicInterval(*locus), i] = array.sum(axis=1)
return garray
datatags = [name] + datatags if datatags else [name]
# At the moment, we treat the information contained
# in each bw-file as unstranded
cover = create_genomic_array(gsize,
stranded=stranded,
storage=storage,
datatags=datatags,
cache=cache,
conditions=conditions,
overwrite=overwrite,
typecode=dtype,
store_whole_genome=store_whole_genome,
resolution=resolution,
loader=_bam_loader,
loader_args=(bamfiles, ))
return cls(name,
cover,
gindexer,
padding_value=0,
dimmode='all',
channel_last=channel_last)
def create_from_array(
cls,
name, # pylint: disable=too-many-locals
array,
gindexer,
genomesize=None,
conditions=None,
resolution=1,
storage='ndarray',
overwrite=False,
datatags=None,
cache=False,
channel_last=True,
store_whole_genome=False):
"""Create a Cover class from a numpy.array.
The purpose of this function is to convert output prediction from
keras which are in numpy.array format into a Cover object.
Parameters
-----------
name : str
Name of the dataset
array : numpy.array
A 4D numpy array that will be re-interpreted as genomic array.
gindexer : GenomicIndexer
Genomic indices associated with the values contained in array.
genomesize : dict or None
Dictionary containing the genome size to fetch the coverage from.
If `genomesize=None`, the genome size is automatically determined
from the GenomicIndexer. If `store_whole_genome=False` this
option does not have an effect.
conditions : list(str) or None
List of conditions.
If `conditions=None`,
the conditions are obtained from
the filenames (without the directories
and file-ending).
resolution : int
Resolution in base pairs divides the region of interest
in windows of length resolution.
This effectively reduces the storage for coverage data.
The resolution must be selected such that min(stepsize, binsize)
is a multiple of resolution.
Default: 1.
storage : str
Storage mode for storing the coverage data can be
'ndarray', 'hdf5' or 'sparse'. Default: 'ndarray'.
overwrite : boolean
Overwrite cachefiles. Default: False.
datatags : list(str) or None
List of datatags. Together with the dataset name,
the datatags are used to construct a cache file.
If :code:`cache=False`, this option does not have an effect.
Default: None.
cache : boolean
Indicates whether to cache the dataset. Default: False.
store_whole_genome : boolean
Indicates whether the whole genome or only selected regions
should be loaded. Default: False.
channel_last : boolean
This tells the constructor how to interpret the array dimensions.
It indicates whether the condition axis is the last dimension
or the first. For example, tensorflow expects the channel at the
last position. Default: True.
"""
if not store_whole_genome:
# if whole genome should not be loaded
gsize = {
_iv_to_str(iv.chrom, iv.start, iv.end): iv.end - iv.start
for iv in gindexer
}
elif genomesize:
gsize = genomesize.copy()
else:
# if not supplied, determine the genome size automatically
# based on the gindexer intervals.
gsize = get_genome_size_from_regions(gindexer)
if not channel_last:
array = np.transpose(array, (0, 3, 1, 2))
if conditions is None:
conditions = ["Cond_{}".format(i) for i in range(array.shape[-1])]
# check if dimensions of gindexer and array match
if len(gindexer) != array.shape[0]:
raise ValueError(
"Data incompatible: "
"The number intervals in gindexer"
" must match the number of datapoints in the array "
"(len(gindexer) != array.shape[0])")
if store_whole_genome:
# in this case the intervals must be non-overlapping
# in order to obtain unambiguous data.
if gindexer.binsize > gindexer.stepsize:
raise ValueError(
"Overlapping intervals: "
"With overlapping intervals the mapping between "
"the array and genomic-array values is ambiguous. "
"Please ensure that binsize <= stepsize.")
# determine the resolution
resolution = gindexer[0].length // array.shape[1]
# determine strandedness
stranded = True if array.shape[2] == 2 else False
def _array_loader(garray, array, gindexer):
print("load from array")
for i, region in enumerate(gindexer):
iv = region
for cond in range(array.shape[-1]):
if stranded:
iv.strand = '+'
garray[iv, cond] = array[i, :, 0, cond].astype(dtype)
iv.strand = '-'
garray[iv, cond] = array[i, :, 1, cond].astype(dtype)
else:
garray[iv, cond] = array[i, :, 0, cond]
return garray
# At the moment, we treat the information contained
# in each bw-file as unstranded
datatags = [name] + datatags if datatags else [name]
datatags += ['resolution{}'.format(resolution)]
cover = create_genomic_array(gsize,
stranded=stranded,
storage=storage,
datatags=datatags,
cache=cache,
conditions=conditions,
resolution=resolution,
overwrite=overwrite,
typecode=array.dtype,
store_whole_genome=store_whole_genome,
loader=_array_loader,
loader_args=(array, gindexer))
return cls(name,
cover,
gindexer,
padding_value=0,
dimmode='all',
channel_last=channel_last)
def __init__(
self,
gsize, # pylint: disable=too-many-locals
stranded=True,
conditions=None,
typecode='d',
datatags=None,
resolution=1,
order=1,
store_whole_genome=True,
cache=None,
padding_value=0.0,
overwrite=False,
loader=None,
collapser=None):
super(SparseGenomicArray,
self).__init__(stranded,
conditions,
typecode,
resolution,
order=order,
padding_value=padding_value,
store_whole_genome=store_whole_genome,
collapser=collapser)
cachefile = _get_cachefile(cache, datatags, '.npz')
load_from_file = _load_data(cache, datatags, '.npz')
gsize_ = None
if not store_whole_genome:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
self.region2index = {_iv_to_str(region.chrom,
region.start,
region.end): i \
for i, region in enumerate(gsize_)}
if load_from_file:
if gsize_ is None:
gsize_ = gsize() if callable(gsize) else gsize
if store_whole_genome:
data = {
str(region.chrom): sparse.dok_matrix(
(_get_iv_length(region.length - self.order + 1,
resolution),
(2 if stranded else 1) * len(self.condition)),
dtype=self.typecode)
for region in gsize_
}
else:
data = {
'data':
sparse.dok_matrix(
(len(gsize_), (_get_iv_length(
gsize_.binsize + 2 * gsize_.flank - self.order +
1, self.resolution)
if self.resolution is not None else 1) *
(2 if stranded else 1) * len(self.condition)),
dtype=self.typecode)
}
self.handle = data
# invoke the loader
if loader:
loader(self)
data = self.handle
data = {chrom: data[chrom].tocoo() for chrom in data}
storage = {chrom: np.column_stack([data[chrom].data,
data[chrom].row,
data[chrom].col]) \
for chrom in data}
for region in gsize_:
if store_whole_genome:
storage[region.chrom + '__length__'] = region.length
names = [name for name in storage]
if cachefile is not None:
np.savez(cachefile, **storage)
if cachefile is not None:
print('reload {}'.format(cachefile))
storage = np.load(cachefile)
names = [name for name in storage if '__length__' not in name]
if store_whole_genome:
self.handle = {
name: sparse.coo_matrix(
(storage[name][:, 0], (storage[name][:, 1].astype('int'),
storage[name][:, 2].astype('int'))),
shape=(_get_iv_length(storage[str(name) + '__length__'],
resolution), (2 if stranded else 1) *
len(self.condition))).tocsr()
for name in names
}
else:
# gsize_ is always available for store_whole_genome=False
self.handle = {
name: sparse.coo_matrix(
(storage[name][:, 0], (storage[name][:, 1].astype('int'),
storage[name][:, 2].astype('int'))),
shape=(len(gsize_),
(_get_iv_length(gsize_.binsize +
2 * gsize_.flank, resolution)
if self.resolution is not None else 1) *
(2 if stranded else 1) *
len(self.condition))).tocsr()
for name in names
}
def create_from_refgenome(cls, name, refgenome, roi=None,
binsize=None,
stepsize=None,
flank=0, order=1,
storage='ndarray',
datatags=None,
cache=False,
overwrite=False,
channel_last=True,
store_whole_genome=False):
"""Create a Bioseq class from a reference genome.
This constructor loads nucleotide sequences from a reference genome.
If regions of interest (ROI) is supplied, only the respective sequences
are loaded, otherwise the entire genome is fetched.
Parameters
-----------
name : str
Name of the dataset
refgenome : str
Fasta file.
roi : str or None
Bed-file defining the region of interest.
If set to None, the sequence will be
fetched from the entire genome and a
genomic indexer must be attached later.
Otherwise, the coverage is only determined
for the region of interest.
binsize : int or None
Binsize in basepairs. For binsize=None,
the binsize will be determined from the bed-file directly
which requires that all intervals in the bed-file are of equal
length. Otherwise, the intervals in the bed-file will be
split to subintervals of length binsize in conjunction with
stepsize. Default: None.
stepsize : int or None
stepsize in basepairs for traversing the genome.
If stepsize is None, it will be set equal to binsize.
Default: None.
flank : int
Flanking region in basepairs to be extended up and downstream of each interval.
Default: 0.
order : int
Order for the one-hot representation. Default: 1.
storage : str
Storage mode for storing the sequence may be 'ndarray', 'hdf5' or
'sparse'. Default: 'hdf5'.
datatags : list(str) or None
List of datatags. Together with the dataset name,
the datatags are used to construct a cache file.
If :code:`cache=False`, this option does not have an effect.
Default: None.
cache : boolean
Indicates whether to cache the dataset. Default: False.
overwrite : boolean
Overwrite the cachefiles. Default: False.
store_whole_genome : boolean
Indicates whether the whole genome or only ROI
should be loaded. If False, a bed-file with regions of interest
must be specified. Default: False.
"""
# fill up int8 rep of DNA
# load bioseq, region index, and within region index
if roi is not None:
gindexer = GenomicIndexer.create_from_file(roi, binsize,
stepsize, flank)
else:
gindexer = None
if not store_whole_genome and gindexer is None:
raise ValueError('Either roi must be supplied or store_whole_genome must be True')
if isinstance(refgenome, str):
seqs = sequences_from_fasta(refgenome, 'dna')
else:
# This is already a list of SeqRecords
seqs = refgenome
if not store_whole_genome and gindexer is not None:
# the genome is loaded with a bed file,
# only the specific subset is loaded
# to keep the memory overhead low.
# Otherwise the entire reference genome is loaded.
rgen = {seq.id: seq for seq in seqs}
subseqs = []
for giv in gindexer:
subseq = rgen[giv.chrom][giv.start:(giv.end)]
subseq.id = _iv_to_str(giv.chrom, giv.start, giv.end - order + 1)
subseq.name = subseq.id
subseq.description = subseq.id
subseqs.append(subseq)
seqs = subseqs
garray = cls._make_genomic_array(name, seqs, order, storage,
datatags=datatags,
cache=cache,
overwrite=overwrite,
store_whole_genome=store_whole_genome)
return cls(name, garray, gindexer,
alphabetsize=len(seqs[0].seq.alphabet.letters),
channel_last=channel_last)
def __setitem__(self, index, value):
interval = index[0]
condition = index[1]
if isinstance(condition, slice) and value.ndim != 3:
raise ValueError('Expected 3D array with condition slice.')
if isinstance(condition, slice):
condition = slice(None, value.shape[-1], None)
if self.stranded and value.shape[1] != 2:
raise ValueError(
'If genomic array is in stranded mode, shape[-1] == 2 is expected'
)
if not self.stranded and value.shape[1] != 1:
value = value.sum(axis=1, keepdims=True)
if isinstance(interval, Interval) and isinstance(
condition, (int, slice)):
chrom = interval.chrom
start = self.get_iv_start(interval.start)
end = self.get_iv_end(interval.end)
# value should be a 2 dimensional array
# it will be reshaped to a 2D array where the collapse operation is performed
# along the second dimension.
if self.collapser is not None:
if self.resolution is None and value.shape[0] == 1 or \
self.resolution is not None and \
value.shape[0] == interval.length//self.resolution:
# collapsing becomes obsolete, because the data has already
# the expected shape (after collapsing)
pass
else:
if self.resolution is None:
# collapse along the entire interval
value = value.reshape((1, ) + value.shape)
else:
# collapse in bins of size resolution
value = value.reshape((
value.shape[0] //
min(self.resolution, value.shape[0]),
min(self.resolution, value.shape[0]),
) + value.shape[1:])
value = self.collapser(value)
try:
if not self._full_genome_stored:
regidx = self.region2index[_iv_to_str(
chrom, interval.start, interval.end)]
nconditions = len(self.condition)
ncondstrand = len(self.condition) * value.shape[-1]
end = end - self.order + 1
idxs = np.where(value > 0)
for idx in zip(*idxs):
basepos = idx[0] * ncondstrand
strand = idx[1] * nconditions
cond = condition if isinstance(condition,
int) else idx[2]
self.handle['data'][regidx, basepos + strand +
cond] = value[idx]
else:
ref_start, ref_end, array_start, \
array_end = self._get_indices(interval, value.shape[0])
idxs = np.where(value > 0)
iarray = np.arange(ref_start, ref_end)
for idx in zip(*idxs):
cond = condition if isinstance(condition,
int) else idx[2]
self.handle[chrom][iarray[idx[0]],
idx[1] * len(self.condition) +
cond] = value[idx[0] +
array_start][idx[1:]]
except KeyError:
# we end up here if the peak regions are not a subset of
# the regions of interest. that might be the case if
# peaks from the holdout proportion of the genome are tried
# to be added.
# unfortunately, it is also possible that store_whole_genome=False
# and the peaks and regions of interest are just not synchronized
# in which case nothing (or too few peaks) are added. in the latter
# case an error would help actually, but I am not sure how to
# check if the first or the second is the case here.
pass
return
raise IndexError("Index must be a Interval and a condition index")
def create_from_bigwig(
cls,
name, # pylint: disable=too-many-locals
bigwigfiles,
regions=None,
genomesize=None,
conditions=None,
binsize=None,
stepsize=None,
resolution=1,
flank=0,
storage='ndarray',
dtype='float32',
overwrite=False,
dimmode='all',
aggregate=np.mean,
datatags=None,
cache=False,
store_whole_genome=False,
channel_last=True,
nan_to_num=True):
"""Create a Cover class from a bigwig-file (or files).
Parameters
-----------
name : str
Name of the dataset
bigwigfiles : str or list
bigwig-file or list of bigwig files.
regions : str or None
Bed-file defining the region of interest.
If set to None, the coverage will be
fetched from the entire genome and a
genomic indexer must be attached later.
Otherwise, the coverage is only determined
for the region of interest.
genomesize : dict or None
Dictionary containing the genome size.
If `genomesize=None`, the genome size
is determined from the bigwig file.
If `store_whole_genome=False`, this option does not have an effect.
conditions : list(str) or None
List of conditions.
If `conditions=None`,
the conditions are obtained from
the filenames (without the directories
and file-ending).
binsize : int or None
Binsize in basepairs. For binsize=None,
the binsize will be determined from the bed-file directly
which requires that all intervals in the bed-file are of equal
length. Otherwise, the intervals in the bed-file will be
split to subintervals of length binsize in conjunction with
stepsize. Default: None.
stepsize : int or None
stepsize in basepairs for traversing the genome.
If stepsize is None, it will be set equal to binsize.
Default: None.
resolution : int
Resolution in base pairs divides the region of interest
in windows of length resolution.
This effectively reduces the storage for coverage data.
The resolution must be selected such that min(stepsize, binsize)
is a multiple of resolution.
Default: 1.
flank : int
Flanking size increases the interval size at both ends by
flank bins. Note that the binsize is defined by the resolution parameter.
Default: 0.
storage : str
Storage mode for storing the coverage data can be
'ndarray', 'hdf5' or 'sparse'. Default: 'ndarray'.
dtype : str
Typecode to define the datatype to be used for storage.
Default: 'float32'.
dimmode : str
Dimension mode can be 'first' or 'all'. If 'first', only
the first element of size resolution is returned. Otherwise,
all elements of size resolution spanning the interval are returned.
Default: 'all'.
overwrite : boolean
Overwrite cachefiles. Default: False.
datatags : list(str) or None
List of datatags. Together with the dataset name,
the datatags are used to construct a cache file.
If :code:`cache=False`, this option does not have an effect.
Default: None.
aggregate : callable
Aggregation operation for loading genomic array.
Default: numpy.mean
cache : boolean
Indicates whether to cache the dataset. Default: False.
store_whole_genome : boolean
Indicates whether the whole genome or only selected regions
should be loaded. If False, a bed-file with regions of interest
must be specified. Default: False.
channel_last : boolean
Indicates whether the condition axis should be the last dimension
or the first. For example, tensorflow expects the channel at the
last position. Default: True.
nan_to_num : boolean
Indicates whether NaN values contained in the bigwig files should
be interpreted as zeros. Default: True
"""
if pyBigWig is None: # pragma: no cover
raise Exception(
'pyBigWig not available. '
'`create_from_bigwig` requires pyBigWig to be installed.')
if regions is not None:
gindexer = GenomicIndexer.create_from_file(regions, binsize,
stepsize, flank)
else:
gindexer = None
if isinstance(bigwigfiles, str):
bigwigfiles = [bigwigfiles]
if not store_whole_genome and not gindexer:
raise ValueError(
'Either regions must be supplied or store_whole_genome must be True'
)
if not store_whole_genome:
# if whole genome should not be loaded
gsize = {
_iv_to_str(iv.chrom, iv.start, iv.end): iv.end - iv.start
for iv in gindexer
}
else:
# otherwise the whole genome will be fetched, or at least
# a set of full length chromosomes
if genomesize is not None:
# if a genome size has specifically been given, use it.
gsize = genomesize.copy()
else:
bwfile = pyBigWig.open(bigwigfiles[0], 'r')
gsize = bwfile.chroms()
if conditions is None:
conditions = [
os.path.splitext(os.path.basename(f))[0] for f in bigwigfiles
]
def _bigwig_loader(garray, aggregate):
print("load from bigwig")
for i, sample_file in enumerate(bigwigfiles):
bwfile = pyBigWig.open(sample_file)
for chrom in gsize:
vals = np.zeros(
(get_chrom_length(gsize[chrom], resolution), ),
dtype=dtype)
locus = _str_to_iv(chrom, template_extension=0)
if len(locus) == 1:
locus = locus + (0, gsize[chrom])
# when only to load parts of the genome
for start in range(locus[1], locus[2], resolution):
if garray._full_genome_stored:
# be careful not to overshoot at the chromosome end
end = min(start + resolution, gsize[chrom])
else:
end = start + resolution
x = np.asarray(
bwfile.values(locus[0], int(start), int(end)))
if nan_to_num:
x = np.nan_to_num(x, copy=False)
vals[(start - locus[1]) // resolution] = aggregate(x)
garray[GenomicInterval(*locus), i] = vals
return garray
datatags = [name] + datatags if datatags else [name]
datatags += ['resolution{}'.format(resolution)]
cover = create_genomic_array(gsize,
stranded=False,
storage=storage,
datatags=datatags,
cache=cache,
conditions=conditions,
overwrite=overwrite,
resolution=resolution,
store_whole_genome=store_whole_genome,
typecode=dtype,
loader=_bigwig_loader,
loader_args=(aggregate, ))
return cls(name,
cover,
gindexer,
padding_value=0,
dimmode=dimmode,
channel_last=channel_last)
