def bam_blacklisted_reads(bam_handle, chroms_to_ignore, blackListFileName=None, numberOfProcessors=1): blacklisted = 0 if blackListFileName is None: return blacklisted # Get the chromosome lengths chromLens = {} lines = pysam.idxstats(bam_handle.filename) lines = toString(lines) if type(lines) is str: lines = lines.strip().split('\n') for line in lines: chrom, _len, nmapped, _nunmapped = line.split('\t') chromLens[chrom] = int(_len) bl = GTF(blackListFileName) regions = [] for chrom in bl.chroms: if (not chroms_to_ignore or chrom not in chroms_to_ignore) and chrom in chromLens: for reg in bl.findOverlaps(chrom, 0, chromLens[chrom]): regions.append([bam_handle.filename, chrom, reg[0], reg[1]]) if len(regions) > 0: import multiprocessing if len(regions) > 1 and numberOfProcessors > 1: pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(bam_blacklisted_worker, regions).get(9999999) else: res = [bam_blacklisted_worker(x) for x in regions] for val in res: blacklisted += val return blacklisted
def bam_blacklisted_reads(bam_handle, chroms_to_ignore, blackListFileName=None, numberOfProcessors=1): blacklisted = 0 if blackListFileName is None: return blacklisted # Get the chromosome lengths chromLens = {} lines = pysam.idxstats(bam_handle.filename) lines = toString(lines) if type(lines) is str: lines = lines.strip().split('\n') for line in lines: chrom, _len, nmapped, _nunmapped = line.split('\t') chromLens[chrom] = int(_len) bl = GTF(blackListFileName) regions = [] for chrom in bl.chroms: if (not chroms_to_ignore or chrom not in chroms_to_ignore) and chrom in chromLens: for reg in bl.findOverlaps(chrom, 0, chromLens[chrom]): regions.append([bam_handle.filename, chrom, reg[0], reg[1]]) if len(regions) > 0: import multiprocessing if len(regions) > 1 and numberOfProcessors > 1: pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(bam_blacklisted_worker, regions).get(9999999) else: res = [bam_blacklisted_worker(x) for x in regions] for val in res: blacklisted += val return blacklisted
def bam_blacklisted_reads(bam_handle, chroms_to_ignore, blackListFileName=None, numberOfProcessors=1): blacklisted = 0 if blackListFileName is None: return blacklisted # Get the chromosome lengths chromLens = {x: y for x, y in zip(bam_handle.references, bam_handle.lengths)} bl = GTF(blackListFileName) hasOverlaps, minOverlap = bl.hasOverlaps(returnDistance=True) if hasOverlaps: sys.exit("Your blacklist file(s) has (have) regions that overlap. Proceeding with such a file would result in deepTools incorrectly calculating scaling factors. As such, you MUST fix this issue before being able to proceed.\n") if minOverlap < 1000: sys.stderr.write("WARNING: The minimum distance between intervals in your blacklist is {}. It makes little biological sense to include small regions between two blacklisted regions. Instead, these should likely be blacklisted as well.\n".format(minOverlap)) regions = [] for chrom in bl.chroms: if (not chroms_to_ignore or chrom not in chroms_to_ignore) and chrom in chromLens: for reg in bl.findOverlaps(chrom, 0, chromLens[chrom]): regions.append([bam_handle.filename, chrom, reg[0], reg[1]]) if len(regions) > 0: import multiprocessing if len(regions) > 1 and numberOfProcessors > 1: pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(bam_blacklisted_worker, regions).get(9999999) else: res = [bam_blacklisted_worker(x) for x in regions] for val in res: blacklisted += val return blacklisted
def getPositionsToSample(chrom, start, end, stepSize): """ check if the region submitted to the worker overlaps with the region to take extra effort to sample. If that is the case, the regions to sample array is increased to match each of the positions in the extra effort region sampled at the same stepSize along the interval. If a filter out tree is given, then from positions to sample those regions are cleaned """ positions_to_sample = np.arange(start, end, stepSize) if global_vars['filter_out']: filter_out_tree = GTF(global_vars['filter_out']) else: filter_out_tree = None if global_vars['extra_sampling_file']: extra_tree = GTF(global_vars['extra_sampling_file']) else: extra_tree = None if extra_tree: orig_len = len(positions_to_sample) try: extra_match = extra_tree.findOverlaps(chrom, start, end) except KeyError: extra_match = [] if len(extra_match) > 0: for intval in extra_match: positions_to_sample = np.append(positions_to_sample, list(range(intval[0], intval[1], stepSize))) # remove duplicates positions_to_sample = np.unique(np.sort(positions_to_sample)) if debug: print("sampling increased to {} from {}".format( len(positions_to_sample), orig_len)) # skip regions that are filtered out if filter_out_tree: try: out_match = filter_out_tree.findOverlaps(chrom, start, end) except KeyError: out_match = [] if len(out_match) > 0: for intval in out_match: positions_to_sample = \ positions_to_sample[ (positions_to_sample < intval[0]) | (positions_to_sample >= intval[1])] return positions_to_sample
def getChunkLength(args, chromSize): """ There's no point in parsing the GTF time over and over again needlessly. Emprically, it seems that adding ~4x the number of workers is ideal, since coverage is non-uniform. This is a heuristic way of approximating that. Note that if there are MANY small contigs and a few large ones (e.g., the max and median lengths are >10x different, then it's best to take a different tack. """ if args.region: chromSize, region_start, region_end, genomeChunkLength = getUserRegion(chromSize, args.region) rv = np.ceil((region_start - region_end) / float(4 * args.numberOfProcessors)).astype(int) return max(1, rv) bl = None if args.blackListFileName: bl = GTF(args.blackListFileName) lengths = [] for k, v in chromSize: regs = blSubtract(bl, k, [0, v]) for reg in regs: lengths.append(reg[1] - reg[0]) if len(lengths) >= 4 * args.numberOfProcessors: rv = np.median(lengths).astype(int) # In cases like dm6 or GRCh38, there are a LOT of really small contigs, which will cause the median to be small and performance to tank if np.max(lengths) >= 10 * rv: rv = np.ceil(np.sum(lengths) / (4.0 * args.numberOfProcessors)).astype(int) else: rv = np.ceil(np.sum(lengths) / (4.0 * args.numberOfProcessors)).astype(int) return max(1, rv)
def makeRegions(BED, args): """ Given a list of BED/GTF files, make a list of regions. These are vaguely extended as appropriate. For simplicity, the maximum of --beforeRegionStartLength and --afterRegionStartLength are tacked on to each end and transcripts are used for GTF files. """ itree = GTF(BED, transcriptID=args.transcriptID, transcript_id_designator=args.transcript_id_designator) o = [] extend = 0 # The before/after stuff is specific to computeMatrix if "beforeRegionsStartLength" in args: extend = max(args.beforeRegionsStartLength, args.afterRegionsStartLength) for chrom in itree.chroms: regs = itree.findOverlaps(chrom, 0, 4294967295) # bigWig files use 32 bit coordinates for reg in regs: o.append([chrom, max(0, reg[0] - extend), reg[1] + extend]) del itree return o
def bam_blacklisted_reads(bam_handle, chroms_to_ignore, blackListFileName=None, numberOfProcessors=1): blacklisted = 0 if blackListFileName is None: return blacklisted # Get the chromosome lengths chromLens = { x: y for x, y in zip(bam_handle.references, bam_handle.lengths) } bl = GTF(blackListFileName) hasOverlaps, minOverlap = bl.hasOverlaps(returnDistance=True) if hasOverlaps: sys.exit( "Your blacklist file(s) has (have) regions that overlap. Proceeding with such a file would result in deepTools incorrectly calculating scaling factors. As such, you MUST fix this issue before being able to proceed.\n" ) if minOverlap < 1000: sys.stderr.write( "WARNING: The minimum distance between intervals in your blacklist is {}. It makes little biological sense to include small regions between two blacklisted regions. Instead, these should likely be blacklisted as well.\n" .format(minOverlap)) regions = [] for chrom in bl.chroms: if (not chroms_to_ignore or chrom not in chroms_to_ignore) and chrom in chromLens: for reg in bl.findOverlaps(chrom, 0, chromLens[chrom]): regions.append([bam_handle.filename, chrom, reg[0], reg[1]]) if len(regions) > 0: import multiprocessing if len(regions) > 1 and numberOfProcessors > 1: pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(bam_blacklisted_worker, regions).get(9999999) else: res = [bam_blacklisted_worker(x) for x in regions] for val in res: blacklisted += val return blacklisted
def makeRegions(BED, args): """ Given a list of BED/GTF files, make a list of regions. These are vaguely extended as appropriate. For simplicity, the maximum of --beforeRegionStartLength and --afterRegionStartLength are tacked on to each end and transcripts are used for GTF files. """ itree = GTF(BED, transcriptID=args.transcriptID, transcript_id_designator=args.transcript_id_designator) o = [] extend = 0 # The before/after stuff is specific to computeMatrix if "beforeRegionStartLength" in args: extend = max(args.beforeRegionStartLength, args.afterRegionStartLength) for chrom in itree.chroms: regs = itree.findOverlaps( chrom, 0, 4294967295) # bigWig files use 32 bit coordinates for reg in regs: o.append([chrom, max(0, reg[0] - extend), reg[1] + extend]) del itree return o
def getPositionsToSample(chrom, start, end, stepSize): """ check if the region submitted to the worker overlaps with the region to take extra effort to sample. If that is the case, the regions to sample array is increased to match each of the positions in the extra effort region sampled at the same stepSize along the interval. If a filter out tree is given, then from positions to sample those regions are cleaned """ positions_to_sample = np.arange(start, end, stepSize) if global_vars['filter_out']: filter_out_tree = GTF(global_vars['filter_out']) else: filter_out_tree = None if global_vars['extra_sampling_file']: extra_tree = GTF(global_vars['extra_sampling_file']) else: extra_tree = None if extra_tree: orig_len = len(positions_to_sample) try: extra_match = extra_tree.findOverlaps(chrom, start, end) except KeyError: extra_match = [] if len(extra_match) > 0: for intval in extra_match: positions_to_sample = np.append(positions_to_sample, list(range(intval[0], intval[1], stepSize))) # remove duplicates positions_to_sample = np.unique(np.sort(positions_to_sample)) if debug: print("sampling increased to {} from {}".format( len(positions_to_sample), orig_len)) # skip regions that are filtered out if filter_out_tree: try: out_match = filter_out_tree.findOverlaps(chrom, start, end) except KeyError: out_match = [] if len(out_match) > 0: for intval in out_match: positions_to_sample = \ positions_to_sample[ (positions_to_sample < intval[0]) | (positions_to_sample >= intval[1])] return positions_to_sample
def main(): parser = argparse.ArgumentParser(add_help=True, description="Bourbon finds contiguous regions without repeats (low peat content) of a minimum size and without genes within some distance. Output is written to the terminal. Note that this program currently ignores the ends of chromosomes.") parser.add_argument("rmsk", help="Repeat masker file") parser.add_argument("gtf", help="GTF file") parser.add_argument("tbit", help="2bit file") parser.add_argument("--minimumProof", type=int, default=15000, help="Minimum size of a repeat-free region (default %(default)s)") parser.add_argument("--wobble", type=int, default=5000, help="Ensure no genes are within this distance of a region of interest (default %(default)s)") parser.add_argument("--legalBAC", type=float, default=0.01, help="Maximum N content (default %(default)s)") args = parser.parse_args() # Produce a header print("Chromosome\tStart\tEnd") genes = GTF(args.gtf) rmsk = open(args.rmsk) tb = py2bit.open(args.tbit) lastChrom = None lastEnd = 0 for line in rmsk: if line.startswith("#"): continue cols = line.strip().split() chrom = cols[5] start = int(cols[6]) - 1 end = int(cols[7]) if chrom == lastChrom: if start - lastEnd >= args.minimumProof: ROIstart = lastEnd ROIend = start blocks = splitByGenes(chrom, ROIstart, ROIend, genes, args.wobble) for block in blocks: if block[1] - block[0] < args.minimumProof: continue if not highN(chrom, block[0], block[1], tb, args.legalBAC): print("{}\t{}\t{}".format(chrom, block[0], block[1])) lastChrom = chrom lastEnd = end rmsk.close() tb.close()
def mapReduce(staticArgs, func, chromSize, genomeChunkLength=None, region=None, bedFile=None, blackListFileName=None, numberOfProcessors=4, verbose=False, includeLabels=False, keepExons=False, transcriptID="transcriptID", exonID="exonID", transcript_id_designator="transcript_id", self_=None): """ Split the genome into parts that are sent to workers using a defined number of procesors. Results are collected and returned. For each genomic region the given 'func' is called using the following parameters: chrom, start, end, staticArgs The *arg* are static, *pickable* variables that need to be sent to workers. The genome chunk length corresponds to a fraction of the genome, in bp, that is send to each of the workers for processing. Depending on the type of process a larger or shorter regions may be preferred :param chromSize: A list of duples containing the chromosome name and its length :param region: The format is chr:start:end:tileSize (see function getUserRegion) :param staticArgs: tuple of arguments that are sent to the given 'func' :param func: function to call. The function is called using the following parameters (chrom, start, end, staticArgs) :param bedFile: Is a bed file is given, the args to the func to be called are extended to include a list of bed defined regions. :param blackListFileName: A list of regions to exclude from all computations. Note that this has genomeChunkLength resolution... :param self_: In case mapreduce should make a call to an object the self variable has to be passed. :param includeLabels: Pass group and transcript labels into the calling function. These are added to the static args (groupLabel and transcriptName). If "includeLabels" is true, a tuple of (results, labels) is returned """ if not genomeChunkLength: genomeChunkLength = 1e5 genomeChunkLength = int(genomeChunkLength) if verbose: print("genome partition size for multiprocessing: {0}".format( genomeChunkLength)) region_start = 0 region_end = None # if a region is set, that means that the task should be only cover # the given genomic position if region: chromSize, region_start, region_end, genomeChunkLength = getUserRegion( chromSize, region) if verbose: print("chrom size: {0}, region start: {1}, region end: {2}, " "genome chunk length sent to each procesor: {3}".format( chromSize, region_start, region_end, genomeChunkLength)) if bedFile: defaultGroup = None if len(bedFile) == 1: defaultGroup = "genes" bed_interval_tree = GTF( bedFile, defaultGroup=defaultGroup, transcriptID=transcriptID, exonID=exonID, transcript_id_designator=transcript_id_designator, keepExons=keepExons) if blackListFileName: blackList = GTF(blackListFileName) TASKS = [] # iterate over all chromosomes for chrom, size in chromSize: # the start is zero unless a specific region is defined start = 0 if region_start == 0 else region_start for startPos in range(start, size, genomeChunkLength): endPos = min(size, startPos + genomeChunkLength) # Reject a chunk if it overlaps if blackListFileName: regions = blSubtract(blackList, chrom, [startPos, endPos]) else: regions = [[startPos, endPos]] for reg in regions: if self_ is not None: argsList = [self_] else: argsList = [] argsList.extend([chrom, reg[0], reg[1]]) # add to argument list the static list received the the function argsList.extend(staticArgs) # if a bed file is given, append to the TASK list, # a list of bed regions that overlap with the # current genomeChunk. if bedFile: # This effectively creates batches of intervals, which is # generally more performant due to the added overhead of # initializing additional workers. # TODO, there's no point in including the chromosome if includeLabels: bed_regions_list = [[ chrom, x[4], x[2], x[3], x[5], x[6] ] for x in bed_interval_tree.findOverlaps( chrom, reg[0], reg[1], trimOverlap=True, numericGroups=True, includeStrand=True)] else: bed_regions_list = [[ chrom, x[4], x[5], x[6] ] for x in bed_interval_tree.findOverlaps( chrom, reg[0], reg[1], trimOverlap=True, includeStrand=True)] if len(bed_regions_list) == 0: continue # add to argument list, the position of the bed regions to use argsList.append(bed_regions_list) TASKS.append(tuple(argsList)) if len(TASKS) > 1 and numberOfProcessors > 1: if verbose: print(("using {} processors for {} " "number of tasks".format(numberOfProcessors, len(TASKS)))) random.shuffle(TASKS) pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(func, TASKS).get(9999999) else: res = list(map(func, TASKS)) if includeLabels: if bedFile: return res, bed_interval_tree.labels else: return res, None return res
def get_coverage_of_region(self, bamHandle, chrom, regions, fragmentFromRead_func=None): """ Returns a numpy array that corresponds to the number of reads that overlap with each tile. >>> test = Tester() >>> import pysam >>> c = CountReadsPerBin([], stepSize=1, extendReads=300) For this case the reads are length 36. The number of overlapping read fragments is 4 and 5 for the positions tested. >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000833, 5000834), (5000834, 5000835)]) array([4., 5.]) In the following example a paired read is extended to the fragment length which is 100 The first mate starts at 5000000 and the second at 5000064. Each mate is extended to the fragment length *independently* At position 500090-500100 one fragment of length 100 overlap, and after position 5000101 there should be zero reads. >>> c.zerosToNans = True >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000090, 5000100), (5000100, 5000110)]) array([ 1., nan]) In the following case the reads length is 50. Reads are not extended. >>> c.extendReads=False >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile2), '3R', [(148, 150), (150, 152), (152, 154)]) array([1., 2., 2.]) """ if not fragmentFromRead_func: fragmentFromRead_func = self.get_fragment_from_read nbins = len(regions) if len(regions[0]) == 3: nbins = 0 for reg in regions: nbins += (reg[1] - reg[0]) // reg[2] coverages = np.zeros(nbins, dtype='float64') if self.defaultFragmentLength == 'read length': extension = 0 else: extension = self.maxPairedFragmentLength blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) vector_start = 0 for idx, reg in enumerate(regions): if len(reg) == 3: tileSize = int(reg[2]) nRegBins = (reg[1] - reg[0]) // tileSize else: nRegBins = 1 tileSize = int(reg[1] - reg[0]) # Blacklisted regions have a coverage of 0 if blackList and blackList.findOverlaps(chrom, reg[0], reg[1]): continue regStart = int(max(0, reg[0] - extension)) regEnd = reg[1] + int(extension) # If alignments are extended and there's a blacklist, ensure that no # reads originating in a blacklist are fetched if blackList and reg[0] > 0 and extension > 0: o = blackList.findOverlaps(chrom, regStart, reg[0]) if o is not None and len(o) > 0: regStart = o[-1][1] o = blackList.findOverlaps(chrom, reg[1], regEnd) if o is not None and len(o) > 0: regEnd = o[0][0] start_time = time.time() # caching seems faster. TODO: profile the function c = 0 if chrom not in bamHandle.references: raise NameError( "chromosome {} not found in bam file".format(chrom)) prev_pos = set() lpos = None # of previous processed read pair for read in bamHandle.fetch(chrom, regStart, regEnd): if read.is_unmapped: continue if self.minMappingQuality and read.mapq < self.minMappingQuality: continue # filter reads based on SAM flag if self.samFlag_include and read.flag & self.samFlag_include != self.samFlag_include: continue if self.samFlag_exclude and read.flag & self.samFlag_exclude != 0: continue # Fragment lengths tLen = deeptools.utilities.getTLen(read) if self.minFragmentLength > 0 and tLen < self.minFragmentLength: continue if self.maxFragmentLength > 0 and tLen > self.maxFragmentLength: continue # get rid of duplicate reads that have same position on each of the # pairs if self.ignoreDuplicates: # Assuming more or less concordant reads, use the fragment bounds, otherwise the start positions if tLen >= 0: s = read.pos e = s + tLen else: s = read.pnext e = s - tLen if read.reference_id != read.next_reference_id: e = read.pnext if lpos is not None and lpos == read.reference_start \ and (s, e, read.next_reference_id, read.is_reverse) in prev_pos: continue if lpos != read.reference_start: prev_pos.clear() lpos = read.reference_start prev_pos.add( (s, e, read.next_reference_id, read.is_reverse)) # since reads can be split (e.g. RNA-seq reads) each part of the # read that maps is called a position block. try: position_blocks = fragmentFromRead_func(read) except TypeError: # the get_fragment_from_read functions returns None in some cases. # Those cases are to be skipped, hence the continue line. continue last_eIdx = None for fragmentStart, fragmentEnd in position_blocks: if fragmentEnd is None or fragmentStart is None: continue fragmentLength = fragmentEnd - fragmentStart if fragmentLength == 0: continue # skip reads that are not in the region being # evaluated. if fragmentEnd <= reg[0] or fragmentStart >= reg[1]: continue if fragmentStart < reg[0]: fragmentStart = reg[0] if fragmentEnd > reg[0] + len(coverages) * tileSize: fragmentEnd = reg[0] + len(coverages) * tileSize sIdx = vector_start + max( (fragmentStart - reg[0]) // tileSize, 0) eIdx = vector_start + min( np.ceil(float(fragmentEnd - reg[0]) / tileSize).astype('int'), nRegBins) if last_eIdx is not None: sIdx = max(last_eIdx, sIdx) if sIdx >= eIdx: continue sIdx = int(sIdx) eIdx = int(eIdx) coverages[sIdx:eIdx] += 1 last_eIdx = eIdx c += 1 if self.verbose: endTime = time.time() print("%s, processing %s (%.1f per sec) reads @ %s:%s-%s" % (multiprocessing.current_process().name, c, c / (endTime - start_time), chrom, reg[0], reg[1])) vector_start += nRegBins # change zeros to NAN if self.zerosToNans: coverages[coverages == 0] = np.nan return coverages
def count_reads_in_region(self, chrom, start, end, bed_regions_list=None): """Counts the reads in each bam file at each 'stepSize' position within the interval (start, end) for a window or bin of size binLength. The stepSize controls the distance between bins. For example, a step size of 20 and a bin size of 20 will create bins next to each other. If the step size is smaller than the bin size the bins will overlap. If a list of bedRegions is given, then the number of reads that overlaps with each region is counted. Parameters ---------- chrom : str Chrom name start : int start coordinate end : int end coordinate bed_regions_list: list List of list of tuples of the form (start, end) corresponding to bed regions to be processed. If not bed file was passed to the object constructor then this list is empty. Returns ------- numpy array The result is a numpy array that as rows each bin and as columns each bam file. Examples -------- Initialize some useful values >>> test = Tester() >>> c = CountReadsPerBin([test.bamFile1, test.bamFile2], 25, 0, stepSize=50) The transpose is used to get better looking numbers. The first line corresponds to the number of reads per bin in the first bamfile. >>> _array, __ = c.count_reads_in_region(test.chrom, 0, 200) >>> _array array([[0., 0.], [0., 1.], [1., 1.], [1., 2.]]) """ if start > end: raise NameError("start %d bigger that end %d" % (start, end)) if self.stepSize is None and bed_regions_list is None: raise ValueError("stepSize is not set!") # array to keep the read counts for the regions subnum_reads_per_bin = [] start_time = time.time() bam_handles = [] for fname in self.bamFilesList: try: bam_handles.append(bamHandler.openBam(fname)) except SystemExit: sys.exit(sys.exc_info()[1]) except: bam_handles.append(pyBigWig.open(fname)) blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) # A list of lists of tuples transcriptsToConsider = [] if bed_regions_list is not None: transcriptsToConsider = [x[1] for x in bed_regions_list] else: if self.stepSize == self.binLength: transcriptsToConsider.append([(start, end, self.binLength)]) else: for i in range(start, end, self.stepSize): if i + self.binLength > end: break if blackList is not None and blackList.findOverlaps( chrom, i, i + self.binLength): continue transcriptsToConsider.append([(i, i + self.binLength)]) if self.save_data: _file = open(deeptools.utilities.getTempFileName(suffix='.bed'), 'w+t') _file_name = _file.name else: _file_name = '' for bam in bam_handles: for trans in transcriptsToConsider: tcov = self.get_coverage_of_region(bam, chrom, trans) if bed_regions_list is not None: subnum_reads_per_bin.append(np.sum(tcov)) else: subnum_reads_per_bin.extend(tcov) subnum_reads_per_bin = np.concatenate([subnum_reads_per_bin]).reshape( -1, len(self.bamFilesList), order='F') if self.save_data: idx = 0 for i, trans in enumerate(transcriptsToConsider): if len(trans[0]) != 3: starts = ",".join([str(x[0]) for x in trans]) ends = ",".join([str(x[1]) for x in trans]) _file.write("\t".join([chrom, starts, ends]) + "\t") _file.write("\t".join( ["{}".format(x) for x in subnum_reads_per_bin[i, :]]) + "\n") else: for exon in trans: for startPos in range(exon[0], exon[1], exon[2]): if idx >= subnum_reads_per_bin.shape[0]: # At the end of chromosomes (or due to blacklisted regions), there are bins smaller than the bin size # Counts there are added to the bin before them, but range() will still try to include them. break _file.write("{0}\t{1}\t{2}\t".format( chrom, startPos, startPos + exon[2])) _file.write("\t".join([ "{}".format(x) for x in subnum_reads_per_bin[idx, :] ]) + "\n") idx += 1 _file.close() if self.verbose: endTime = time.time() rows = subnum_reads_per_bin.shape[0] print("%s countReadsInRegions_worker: processing %d " "(%.1f per sec) @ %s:%s-%s" % (multiprocessing.current_process().name, rows, rows / (endTime - start_time), chrom, start, end)) return subnum_reads_per_bin, _file_name
def get_coverage_of_region(self, bamHandle, chrom, regions, fragmentFromRead_func=None): """ Returns a numpy array that corresponds to the number of reads that overlap with each tile. >>> test = Tester() >>> import pysam >>> c = SumCoveragePerBin([], stepSize=1, extendReads=300) For this case the reads are length 36. The number of overlapping read fragments is 4 and 5 for the positions tested. Note that reads are NOT extended, due to there being a 0 length input list of BAM files! >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000833, 5000834), (5000834, 5000835)]) array([4., 5.]) In the following case the reads length is 50. Reads are not extended. >>> c.extendReads=False >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile2), '3R', [(148, 150), (150, 152), (152, 154)]) array([2., 4., 4.]) """ if not fragmentFromRead_func: fragmentFromRead_func = self.get_fragment_from_read nbins = len(regions) if len(regions[0]) == 3: nbins = 0 for reg in regions: nbins += (reg[1] - reg[0]) // reg[2] coverages = np.zeros(nbins, dtype='float64') if self.defaultFragmentLength == 'read length': extension = 0 else: extension = self.maxPairedFragmentLength blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) vector_start = 0 for idx, reg in enumerate(regions): if len(reg) == 3: tileSize = int(reg[2]) nRegBins = (reg[1] - reg[0]) // tileSize else: nRegBins = 1 tileSize = int(reg[1] - reg[0]) # Blacklisted regions have a coverage of 0 if blackList and blackList.findOverlaps(chrom, reg[0], reg[1]): continue regStart = int(max(0, reg[0] - extension)) regEnd = reg[1] + int(extension) # If alignments are extended and there's a blacklist, ensure that no # reads originating in a blacklist are fetched if blackList and reg[0] > 0 and extension > 0: o = blackList.findOverlaps(chrom, regStart, reg[0]) if o is not None and len(o) > 0: regStart = o[-1][1] o = blackList.findOverlaps(chrom, reg[1], regEnd) if o is not None and len(o) > 0: regEnd = o[0][0] start_time = time.time() # caching seems faster. TODO: profile the function c = 0 try: # BAM input if chrom not in bamHandle.references: raise NameError("chromosome {} not found in bam file".format(chrom)) except: # bigWig input, as used by plotFingerprint if bamHandle.chroms(chrom): _ = np.array(bamHandle.stats(chrom, regStart, regEnd, type="mean", nBins=nRegBins), dtype=np.float) _[np.isnan(_)] = 0.0 _ = _ * tileSize coverages += _ continue else: raise NameError("chromosome {} not found in bigWig file with chroms {}".format(chrom, bamHandle.chroms())) prev_pos = set() lpos = None # of previous processed read pair for read in bamHandle.fetch(chrom, regStart, regEnd): if read.is_unmapped: continue if self.minMappingQuality and read.mapq < self.minMappingQuality: continue # filter reads based on SAM flag if self.samFlag_include and read.flag & self.samFlag_include != self.samFlag_include: continue if self.samFlag_exclude and read.flag & self.samFlag_exclude != 0: continue # Fragment lengths tLen = getTLen(read) if self.minFragmentLength > 0 and tLen < self.minFragmentLength: continue if self.maxFragmentLength > 0 and tLen > self.maxFragmentLength: continue # get rid of duplicate reads that have same position on each of the # pairs if self.ignoreDuplicates: # Assuming more or less concordant reads, use the fragment bounds, otherwise the start positions if tLen >= 0: s = read.pos e = s + tLen else: s = read.pnext e = s - tLen if read.reference_id != read.next_reference_id: e = read.pnext if lpos is not None and lpos == read.reference_start \ and (s, e, read.next_reference_id, read.is_reverse) in prev_pos: continue if lpos != read.reference_start: prev_pos.clear() lpos = read.reference_start prev_pos.add((s, e, read.next_reference_id, read.is_reverse)) # since reads can be split (e.g. RNA-seq reads) each part of the # read that maps is called a position block. try: position_blocks = fragmentFromRead_func(read) except TypeError: # the get_fragment_from_read functions returns None in some cases. # Those cases are to be skipped, hence the continue line. continue last_eIdx = None for fragmentStart, fragmentEnd in position_blocks: if fragmentEnd is None or fragmentStart is None: continue fragmentLength = fragmentEnd - fragmentStart if fragmentLength == 0: continue # skip reads that are not in the region being # evaluated. if fragmentEnd <= reg[0] or fragmentStart >= reg[1]: continue if fragmentStart < reg[0]: fragmentStart = reg[0] if fragmentEnd > reg[0] + len(coverages) * tileSize: fragmentEnd = reg[0] + len(coverages) * tileSize sIdx = vector_start + max((fragmentStart - reg[0]) // tileSize, 0) eIdx = vector_start + min(np.ceil(float(fragmentEnd - reg[0]) / tileSize).astype('int'), nRegBins) if eIdx >= len(coverages): eIdx = len(coverages) - 1 if last_eIdx is not None: sIdx = max(last_eIdx, sIdx) if sIdx >= eIdx: continue # First bin if fragmentEnd < reg[0] + (sIdx + 1) * tileSize: _ = fragmentEnd - fragmentStart else: _ = reg[0] + (sIdx + 1) * tileSize - fragmentStart if _ > tileSize: _ = tileSize coverages[sIdx] += _ _ = sIdx + 1 while _ < eIdx: coverages[_] += tileSize _ += 1 while eIdx - sIdx >= nRegBins: eIdx -= 1 if eIdx > sIdx: _ = fragmentEnd - (reg[0] + eIdx * tileSize) if _ > tileSize: _ = tileSize elif _ < 0: _ = 0 coverages[eIdx] += _ last_eIdx = eIdx c += 1 if self.verbose: endTime = time.time() print("%s, processing %s (%.1f per sec) reads @ %s:%s-%s" % ( multiprocessing.current_process().name, c, c / (endTime - start_time), chrom, reg[0], reg[1])) vector_start += nRegBins # change zeros to NAN if self.zerosToNans: coverages[coverages == 0] = np.nan return coverages
def get_coverage_of_region(self, bamHandle, chrom, regions, fragmentFromRead_func=None): """ Returns a numpy array that corresponds to the number of reads that overlap with each tile. >>> test = Tester() >>> import pysam >>> c = CountReadsPerBin([], stepSize=1, extendReads=300) For this case the reads are length 36. The number of overlapping read fragments is 4 and 5 for the positions tested. >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000833, 5000834), (5000834, 5000835)]) array([ 4., 5.]) In the following example a paired read is extended to the fragment length which is 100 The first mate starts at 5000000 and the second at 5000064. Each mate is extended to the fragment length *independently* At position 500090-500100 one fragment of length 100 overlap, and after position 5000101 there should be zero reads. >>> c.zerosToNans = True >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000090, 5000100), (5000100, 5000110)]) array([ 1., nan]) In the following case the reads length is 50. Reads are not extended. >>> c.extendReads=False >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile2), '3R', [(148, 150), (150, 152), (152, 154)]) array([ 1., 2., 2.]) """ if not fragmentFromRead_func: fragmentFromRead_func = self.get_fragment_from_read nbins = len(regions) if len(regions[0]) == 3: nbins = 0 for reg in regions: nbins += (reg[1] - reg[0]) // reg[2] coverages = np.zeros(nbins, dtype='float64') if self.defaultFragmentLength == 'read length': extension = 0 else: extension = self.maxPairedFragmentLength blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) vector_start = 0 for idx, reg in enumerate(regions): if len(reg) == 3: tileSize = int(reg[2]) nRegBins = (reg[1] - reg[0]) // tileSize else: nRegBins = 1 tileSize = int(reg[1] - reg[0]) # Blacklisted regions have a coverage of 0 if blackList and blackList.findOverlaps(chrom, reg[0], reg[1]): continue regStart = int(max(0, reg[0] - extension)) regEnd = reg[1] + int(extension) # If alignments are extended and there's a blacklist, ensure that no # reads originating in a blacklist are fetched if blackList and reg[0] > 0 and extension > 0: o = blackList.findOverlaps(chrom, regStart, reg[0]) if o is not None and len(o) > 0: regStart = o[-1][1] o = blackList.findOverlaps(chrom, reg[1], regEnd) if o is not None and len(o) > 0: regEnd = o[0][0] start_time = time.time() # caching seems faster. TODO: profile the function c = 0 if chrom in bamHandle.references: reads = [r for r in bamHandle.fetch(chrom, regStart, regEnd) if r.flag & 4 == 0] else: raise NameError("chromosome {} not found in bam file".format(chrom)) prev_start_pos = None # to store the start positions # of previous processed read pair for read in reads: if self.minMappingQuality and read.mapq < self.minMappingQuality: continue # filter reads based on SAM flag if self.samFlag_include and read.flag & self.samFlag_include != self.samFlag_include: continue if self.samFlag_exclude and read.flag & self.samFlag_exclude != 0: continue # Fragment lengths tLen = deeptools.utilities.getTLen(read) if self.minFragmentLength > 0 and tLen < self.minFragmentLength: continue if self.maxFragmentLength > 0 and tLen > self.maxFragmentLength: continue # get rid of duplicate reads that have same position on each of the # pairs if self.ignoreDuplicates and prev_start_pos \ and prev_start_pos == (read.reference_start, read.pnext, read.is_reverse): continue # since reads can be split (e.g. RNA-seq reads) each part of the # read that maps is called a position block. try: position_blocks = fragmentFromRead_func(read) except TypeError: # the get_fragment_from_read functions returns None in some cases. # Those cases are to be skipped, hence the continue line. continue last_eIdx = None for fragmentStart, fragmentEnd in position_blocks: if fragmentEnd is None or fragmentStart is None: continue fragmentLength = fragmentEnd - fragmentStart if fragmentLength == 0: continue # skip reads that are not in the region being # evaluated. if fragmentEnd <= reg[0] or fragmentStart >= reg[1]: continue if fragmentStart < reg[0]: fragmentStart = reg[0] if fragmentEnd > reg[0] + len(coverages) * tileSize: fragmentEnd = reg[0] + len(coverages) * tileSize sIdx = vector_start + max((fragmentStart - reg[0]) // tileSize, 0) eIdx = vector_start + min(np.ceil(float(fragmentEnd - reg[0]) / tileSize).astype('int'), nRegBins) if last_eIdx is not None: sIdx = max(last_eIdx, sIdx) if sIdx >= eIdx: continue coverages[sIdx:eIdx] += 1 last_eIdx = eIdx prev_start_pos = (read.reference_start, read.pnext, read.is_reverse) c += 1 if self.verbose: endTime = time.time() print("%s, processing %s (%.1f per sec) reads @ %s:%s-%s" % ( multiprocessing.current_process().name, c, c / (endTime - start_time), chrom, reg[0], reg[1])) vector_start += nRegBins # change zeros to NAN if self.zerosToNans: coverages[coverages == 0] = np.nan return coverages
def count_reads_in_region(self, chrom, start, end, bed_regions_list=None): """Counts the reads in each bam file at each 'stepSize' position within the interval (start, end) for a window or bin of size binLength. The stepSize controls the distance between bins. For example, a step size of 20 and a bin size of 20 will create bins next to each other. If the step size is smaller than the bin size the bins will overlap. If a list of bedRegions is given, then the number of reads that overlaps with each region is counted. Parameters ---------- chrom : str Chrom name start : int start coordinate end : int end coordinate bed_regions_list: list List of list of tuples of the form (start, end) corresponding to bed regions to be processed. If not bed file was passed to the object constructor then this list is empty. Returns ------- numpy array The result is a numpy array that as rows each bin and as columns each bam file. Examples -------- Initialize some useful values >>> test = Tester() >>> c = CountReadsPerBin([test.bamFile1, test.bamFile2], 25, 0, stepSize=50) The transpose is used to get better looking numbers. The first line corresponds to the number of reads per bin in the first bamfile. >>> _array, __ = c.count_reads_in_region(test.chrom, 0, 200) >>> _array array([[ 0., 0.], [ 0., 1.], [ 1., 1.], [ 1., 2.]]) """ if start > end: raise NameError("start %d bigger that end %d" % (start, end)) if self.stepSize is None: raise ValueError("stepSize is not set!") # array to keep the read counts for the regions subnum_reads_per_bin = [] start_time = time.time() bam_handlers = [] for fname in self.bamFilesList: try: bam_handlers.append(bamHandler.openBam(fname)) except: bam_handlers.append(pyBigWig.open(fname)) blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) # A list of lists of tuples transcriptsToConsider = [] if bed_regions_list is not None: transcriptsToConsider = [x[1] for x in bed_regions_list] else: if self.stepSize == self.binLength: transcriptsToConsider.append([(start, end, self.binLength)]) else: for i in range(start, end, self.stepSize): if i + self.binLength > end: break if blackList is not None and blackList.findOverlaps(chrom, i, i + self.binLength): continue transcriptsToConsider.append([(i, i + self.binLength)]) if self.save_data: _file = open(deeptools.utilities.getTempFileName(suffix='.bed'), 'w+t') _file_name = _file.name else: _file_name = '' for bam in bam_handlers: for trans in transcriptsToConsider: tcov = self.get_coverage_of_region(bam, chrom, trans) if bed_regions_list is not None: subnum_reads_per_bin.append(np.sum(tcov)) else: subnum_reads_per_bin.extend(tcov) subnum_reads_per_bin = np.concatenate([subnum_reads_per_bin]).reshape(-1, len(self.bamFilesList), order='F') if self.save_data: idx = 0 for i, trans in enumerate(transcriptsToConsider): if len(trans[0]) != 3: starts = ",".join([str(x[0]) for x in trans]) ends = ",".join([str(x[1]) for x in trans]) _file.write("\t".join([chrom, starts, ends]) + "\t") _file.write("\t".join(["{}".format(x) for x in subnum_reads_per_bin[i, :]]) + "\n") else: for exon in trans: for startPos in range(exon[0], exon[1], exon[2]): if idx >= subnum_reads_per_bin.shape[0]: # At the end of chromosomes (or due to blacklisted regions), there are bins smaller than the bin size # Counts there are added to the bin before them, but range() will still try to include them. break _file.write("{0}\t{1}\t{2}\t".format(chrom, startPos, startPos + exon[2])) _file.write("\t".join(["{}".format(x) for x in subnum_reads_per_bin[idx, :]]) + "\n") idx += 1 _file.close() if self.verbose: endTime = time.time() rows = subnum_reads_per_bin.shape[0] print("%s countReadsInRegions_worker: processing %d " "(%.1f per sec) @ %s:%s-%s" % (multiprocessing.current_process().name, rows, rows / (endTime - start_time), chrom, start, end)) return subnum_reads_per_bin, _file_name
def count_reads_in_region_with_intron(self, chrom, start, end, bed_regions_list=None): """ Rewrite deeptools.CountReadsPerBin.count_reads_in_region Args: chrom (str): Chrom start (int): Start position end (int): End position bed_regions_list (list): List of bed region Returns: tuple: subnum_reads_per_bin, file_name """ if start > end: raise NameError("start %d bigger that end %d" % (start, end)) if self.stepSize is None: raise ValueError("stepSize is not set!") # array to keep the read counts for the regions subnum_reads_per_bin = [] start_time = time.time() bam_handlers = [] for fname in self.bamFilesList: try: bam_handlers.append(bamHandler.openBam(fname)) except: bam_handlers.append(pyBigWig.open(fname)) blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) # A list of lists of tuples transcriptsToConsider = [] if bed_regions_list is not None: transcriptsToConsider = [x[1] for x in bed_regions_list] else: if self.stepSize == self.binLength: transcriptsToConsider.append([(start, end, self.binLength)]) else: for i in range(start, end, self.stepSize): if i + self.binLength > end: break if blackList is not None and blackList.findOverlaps( chrom, i, i + self.binLength): continue transcriptsToConsider.append([(i, i + self.binLength)]) if self.save_data: _file = open(deeptools.utilities.getTempFileName(suffix='.bed'), 'w+t') _file_name = _file.name else: _file_name = '' for bam in bam_handlers: for trans in transcriptsToConsider: tcov = self.get_coverage_of_region_with_intron( bam, chrom, trans) if bed_regions_list is not None: subnum_reads_per_bin.append(np.sum(tcov)) else: subnum_reads_per_bin.extend(tcov) subnum_reads_per_bin = np.concatenate([subnum_reads_per_bin]).reshape( -1, len(self.bamFilesList), order='F') if self.save_data: idx = 0 for i, trans in enumerate(transcriptsToConsider): if len(trans[0]) != 3: starts = ",".join([str(x[0]) for x in trans]) ends = ",".join([str(x[1]) for x in trans]) _file.write("\t".join([chrom, starts, ends]) + "\t") _file.write("\t".join( ["{}".format(x) for x in subnum_reads_per_bin[i, :]]) + "\n") else: for exon in trans: for startPos in range(exon[0], exon[1], exon[2]): if idx >= subnum_reads_per_bin.shape[0]: # At the end of chromosomes (or due to blacklisted regions), there are bins smaller than the bin size # Counts there are added to the bin before them, but range() will still try to include them. break _file.write("{0}\t{1}\t{2}\t".format( chrom, startPos, startPos + exon[2])) _file.write("\t".join([ "{}".format(x) for x in subnum_reads_per_bin[idx, :] ]) + "\n") idx += 1 _file.close() if self.verbose: endTime = time.time() rows = subnum_reads_per_bin.shape[0] print("%s countReadsInRegions_worker: processing %d " "(%.1f per sec) @ %s:%s-%s" % (multiprocessing.current_process().name, rows, rows / (endTime - start_time), chrom, start, end)) return subnum_reads_per_bin, _file_name
def get_coverage_of_region_with_intron(self, bamHandle, chrom, regions, fragmentFromRead_func=None): """ Rewrite deeptools.CountReadsPerBin.get_coverage_of_region Args: bamHandle (AlignmentFile): Bam object chrom (str): Chrom regions (list): List of block fragmentFromRead_func (function): Function to get fragment from read Returns: float: coverages """ if not fragmentFromRead_func: fragmentFromRead_func = self.get_fragment_from_read nbins = len(regions) if len(regions[0]) == 3: nbins = 0 for reg in regions: nbins += (reg[1] - reg[0]) // reg[2] coverages = np.zeros(nbins, dtype='float64') if self.defaultFragmentLength == 'read length': extension = 0 else: extension = self.maxPairedFragmentLength blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) vector_start = 0 for idx, reg in enumerate(regions): if len(reg) == 3: tileSize = int(reg[2]) nRegBins = (reg[1] - reg[0]) // tileSize else: nRegBins = 1 tileSize = int(reg[1] - reg[0]) # Blacklisted regions have a coverage of 0 if blackList and blackList.findOverlaps(chrom, reg[0], reg[1]): continue regStart = int(max(0, reg[0] - extension)) regEnd = reg[1] + int(extension) # If alignments are extended and there's a blacklist, ensure that no # reads originating in a blacklist are fetched if blackList and reg[0] > 0 and extension > 0: o = blackList.findOverlaps(chrom, regStart, reg[0]) if o is not None and len(o) > 0: regStart = o[-1][1] o = blackList.findOverlaps(chrom, reg[1], regEnd) if o is not None and len(o) > 0: regEnd = o[0][0] start_time = time.time() # caching seems faster. c = 0 if chrom in bamHandle.references: reads = [ r for r in bamHandle.fetch(chrom, regStart, regEnd) if r.flag & 4 == 0 ] else: raise NameError( "chromosome {} not found in bam file".format(chrom)) prev_start_pos = None # to store the start positions # of previous processed read pair for read in reads: if self.minMappingQuality and read.mapq < self.minMappingQuality: continue # filter reads based on SAM flag if self.samFlag_include and read.flag & self.samFlag_include != self.samFlag_include: continue if self.samFlag_exclude and read.flag & self.samFlag_exclude != 0: continue # Fragment lengths tLen = deeptools.utilities.getTLen(read) if self.minFragmentLength > 0 and tLen < self.minFragmentLength: continue if self.maxFragmentLength > 0 and tLen > self.maxFragmentLength: continue # get rid of duplicate reads that have same position on each of the # pairs if self.ignoreDuplicates and prev_start_pos \ and prev_start_pos == (read.reference_start, read.pnext, read.is_reverse): continue # since reads can be split (e.g. RNA-seq reads) each part of the # read that maps is called a position block. try: position_blocks = fragmentFromRead_func(read) except TypeError: # the get_fragment_from_read functions returns None in some cases. # Those cases are to be skipped, hence the continue line. continue # Rewrite !!! sIdx = vector_start + max( (read.reference_start - reg[0]) // tileSize, 0) eIdx = vector_start + min( np.ceil(float(read.reference_end - reg[0]) / tileSize).astype('int'), nRegBins) coverages[sIdx:eIdx] += 1 prev_start_pos = (read.reference_start, read.pnext, read.is_reverse) c += 1 if self.verbose: endTime = time.time() print("%s, processing %s (%.1f per sec) reads @ %s:%s-%s" % (multiprocessing.current_process().name, c, c / (endTime - start_time), chrom, reg[0], reg[1])) vector_start += nRegBins # change zeros to NAN if self.zerosToNans: coverages[coverages == 0] = np.nan return coverages
def get_coverage_of_region(self, bamHandle, chrom, regions, fragmentFromRead_func=None): """ Returns a numpy array that corresponds to the number of reads that overlap with each tile. >>> test = Tester() >>> import pysam >>> c = SumCoveragePerBin([], stepSize=1, extendReads=300) For this case the reads are length 36. The number of overlapping read fragments is 4 and 5 for the positions tested. Note that reads are NOT extended, due to there being a 0 length input list of BAM files! >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile_PE), 'chr2', ... [(5000833, 5000834), (5000834, 5000835)]) array([ 4., 5.]) In the following case the reads length is 50. Reads are not extended. >>> c.extendReads=False >>> c.get_coverage_of_region(pysam.AlignmentFile(test.bamFile2), '3R', [(148, 150), (150, 152), (152, 154)]) array([ 2., 4., 4.]) """ if not fragmentFromRead_func: fragmentFromRead_func = self.get_fragment_from_read nbins = len(regions) if len(regions[0]) == 3: nbins = 0 for reg in regions: nbins += (reg[1] - reg[0]) // reg[2] coverages = np.zeros(nbins, dtype='float64') if self.defaultFragmentLength == 'read length': extension = 0 else: extension = self.maxPairedFragmentLength blackList = None if self.blackListFileName is not None: blackList = GTF(self.blackListFileName) vector_start = 0 for idx, reg in enumerate(regions): if len(reg) == 3: tileSize = int(reg[2]) nRegBins = (reg[1] - reg[0]) // tileSize else: nRegBins = 1 tileSize = int(reg[1] - reg[0]) # Blacklisted regions have a coverage of 0 if blackList and blackList.findOverlaps(chrom, reg[0], reg[1]): continue regStart = int(max(0, reg[0] - extension)) regEnd = reg[1] + int(extension) # If alignments are extended and there's a blacklist, ensure that no # reads originating in a blacklist are fetched if blackList and reg[0] > 0 and extension > 0: o = blackList.findOverlaps(chrom, regStart, reg[0]) if o is not None and len(o) > 0: regStart = o[-1][1] o = blackList.findOverlaps(chrom, reg[1], regEnd) if o is not None and len(o) > 0: regEnd = o[0][0] start_time = time.time() # caching seems faster. TODO: profile the function c = 0 try: # BAM input if chrom in bamHandle.references: reads = [ r for r in bamHandle.fetch(chrom, regStart, regEnd) if r.flag & 4 == 0 ] else: raise NameError( "chromosome {} not found in bam file".format(chrom)) except: # bigWig input, as used by plotFingerprint if bamHandle.chroms(chrom): _ = np.array(bamHandle.stats(chrom, regStart, regEnd, type="mean", nBins=nRegBins), dtype=np.float) _[np.isnan(_)] = 0.0 _ = _ * tileSize coverages += _ continue else: raise NameError( "chromosome {} not found in bigWig file with chroms {}" .format(chrom, bamHandle.chroms())) prev_start_pos = None # to store the start positions # of previous processed read pair for read in reads: if self.minMappingQuality and read.mapq < self.minMappingQuality: continue # filter reads based on SAM flag if self.samFlag_include and read.flag & self.samFlag_include != self.samFlag_include: continue if self.samFlag_exclude and read.flag & self.samFlag_exclude != 0: continue # Fragment lengths if self.minFragmentLength > 0 and abs( read.template_length) < self.minFragmentLength: continue if self.maxFragmentLength > 0 and abs( read.template_length) > self.maxFragmentLength: continue # get rid of duplicate reads that have same position on each of the # pairs if self.ignoreDuplicates and prev_start_pos \ and prev_start_pos == (read.reference_start, read.pnext, read.is_reverse): continue # since reads can be split (e.g. RNA-seq reads) each part of the # read that maps is called a position block. try: position_blocks = fragmentFromRead_func(read) except TypeError: # the get_fragment_from_read functions returns None in some cases. # Those cases are to be skipped, hence the continue line. continue last_eIdx = None for fragmentStart, fragmentEnd in position_blocks: if fragmentEnd is None or fragmentStart is None: continue fragmentLength = fragmentEnd - fragmentStart if fragmentLength == 0: continue # skip reads that are not in the region being # evaluated. if fragmentEnd <= reg[0] or fragmentStart >= reg[1]: continue if fragmentStart < reg[0]: fragmentStart = reg[0] if fragmentEnd > reg[0] + len(coverages) * tileSize: fragmentEnd = reg[0] + len(coverages) * tileSize sIdx = vector_start + max( (fragmentStart - reg[0]) // tileSize, 0) eIdx = vector_start + min( np.ceil(float(fragmentEnd - reg[0]) / tileSize).astype('int'), nRegBins) if eIdx >= len(coverages): eIdx = len(coverages) - 1 if last_eIdx is not None: sIdx = max(last_eIdx, sIdx) if sIdx >= eIdx: continue # First bin if fragmentEnd < reg[0] + (sIdx + 1) * tileSize: _ = fragmentEnd - fragmentStart else: _ = reg[0] + (sIdx + 1) * tileSize - fragmentStart if _ > tileSize: _ = tileSize coverages[sIdx] += _ _ = sIdx + 1 while _ < eIdx: coverages[_] += tileSize _ += 1 while eIdx - sIdx >= nRegBins: eIdx -= 1 if eIdx > sIdx: _ = fragmentEnd - (reg[0] + eIdx * tileSize) if _ > tileSize: _ = tileSize elif _ < 0: _ = 0 coverages[eIdx] += _ last_eIdx = eIdx prev_start_pos = (read.reference_start, read.pnext, read.is_reverse) c += 1 if self.verbose: endTime = time.time() print("%s, processing %s (%.1f per sec) reads @ %s:%s-%s" % (multiprocessing.current_process().name, c, c / (endTime - start_time), chrom, reg[0], reg[1])) vector_start += nRegBins # change zeros to NAN if self.zerosToNans: coverages[coverages == 0] = np.nan return coverages
def mapReduce(staticArgs, func, chromSize, genomeChunkLength=None, region=None, bedFile=None, blackListFileName=None, numberOfProcessors=4, verbose=False, includeLabels=False, keepExons=False, transcriptID="transcriptID", exonID="exonID", transcript_id_designator="transcript_id", self_=None): """ Split the genome into parts that are sent to workers using a defined number of procesors. Results are collected and returned. For each genomic region the given 'func' is called using the following parameters: chrom, start, end, staticArgs The *arg* are static, *pickable* variables that need to be sent to workers. The genome chunk length corresponds to a fraction of the genome, in bp, that is send to each of the workers for processing. Depending on the type of process a larger or shorter regions may be preferred :param chromSize: A list of duples containing the chromosome name and its length :param region: The format is chr:start:end:tileSize (see function getUserRegion) :param staticArgs: tuple of arguments that are sent to the given 'func' :param func: function to call. The function is called using the following parameters (chrom, start, end, staticArgs) :param bedFile: Is a bed file is given, the args to the func to be called are extended to include a list of bed defined regions. :param blackListFileName: A list of regions to exclude from all computations. Note that this has genomeChunkLength resolution... :param self_: In case mapreduce should make a call to an object the self variable has to be passed. :param includeLabels: Pass group and transcript labels into the calling function. These are added to the static args (groupLabel and transcriptName). If "includeLabels" is true, a tuple of (results, labels) is returned """ if not genomeChunkLength: genomeChunkLength = 1e5 genomeChunkLength = int(genomeChunkLength) if verbose: print("genome partition size for multiprocessing: {0}".format( genomeChunkLength)) region_start = 0 region_end = None # if a region is set, that means that the task should only cover # the given genomic position if region: chromSize, region_start, region_end, genomeChunkLength = getUserRegion(chromSize, region) if verbose: print("chrom size: {0}, region start: {1}, region end: {2}, " "genome chunk length sent to each procesor: {3}".format(chromSize, region_start, region_end, genomeChunkLength)) if bedFile: defaultGroup = None if len(bedFile) == 1: defaultGroup = "genes" bed_interval_tree = GTF(bedFile, defaultGroup=defaultGroup, transcriptID=transcriptID, exonID=exonID, transcript_id_designator=transcript_id_designator, keepExons=keepExons) if blackListFileName: blackList = GTF(blackListFileName) TASKS = [] # iterate over all chromosomes for chrom, size in chromSize: # the start is zero unless a specific region is defined start = 0 if region_start == 0 else region_start for startPos in range(start, size, genomeChunkLength): endPos = min(size, startPos + genomeChunkLength) # Reject a chunk if it overlaps if blackListFileName: regions = blSubtract(blackList, chrom, [startPos, endPos]) else: regions = [[startPos, endPos]] for reg in regions: if self_ is not None: argsList = [self_] else: argsList = [] argsList.extend([chrom, reg[0], reg[1]]) # add to argument list the static list received the the function argsList.extend(staticArgs) # if a bed file is given, append to the TASK list, # a list of bed regions that overlap with the # current genomeChunk. if bedFile: # This effectively creates batches of intervals, which is # generally more performant due to the added overhead of # initializing additional workers. # TODO, there's no point in including the chromosome if includeLabels: bed_regions_list = [[chrom, x[4], x[2], x[3], x[5], x[6]] for x in bed_interval_tree.findOverlaps(chrom, reg[0], reg[1], trimOverlap=True, numericGroups=True, includeStrand=True)] else: bed_regions_list = [[chrom, x[4], x[5], x[6]] for x in bed_interval_tree.findOverlaps(chrom, reg[0], reg[1], trimOverlap=True, includeStrand=True)] if len(bed_regions_list) == 0: continue # add to argument list, the position of the bed regions to use argsList.append(bed_regions_list) TASKS.append(tuple(argsList)) if len(TASKS) > 1 and numberOfProcessors > 1: if verbose: print(("using {} processors for {} " "number of tasks".format(numberOfProcessors, len(TASKS)))) random.shuffle(TASKS) pool = multiprocessing.Pool(numberOfProcessors) res = pool.map_async(func, TASKS).get(9999999) else: res = list(map(func, TASKS)) if includeLabels: if bedFile: return res, bed_interval_tree.labels else: return res, None return res