def build_genome_alignment_from_file(ga_path, ref_spec, idx_path=None, verbose=False): """ build a genome alignment by loading from a single MAF file. :param ga_path: the path to the file to load. :param ref_spec: which species in the MAF file is the reference? :param idx_path: if provided, use this index to generate a just-in-time genome alignment, instead of loading the file immediately. """ blocks = [] if (idx_path is not None): bound_iter = functools.partial(genome_alignment_iterator, reference_species=ref_spec) hash_func = JustInTimeGenomeAlignmentBlock.build_hash factory = IndexedFile(None, bound_iter, hash_func) factory.read_index(idx_path, ga_path, verbose=verbose) pind = None for k in factory: if verbose: if pind is None: total = len(factory) pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="building alignment blocks ") pind.done += 1 pind.showProgress() blocks.append(JustInTimeGenomeAlignmentBlock(factory, k)) else: for b in genome_alignment_iterator(ga_path, ref_spec, verbose=verbose): blocks.append(b) return GenomeAlignment(blocks, verbose)
def fixedWigIterator(fd, verbose=False, sortedby=None, scoreType=int): """ @summary: """ fh = openFD(fd) if verbose: try: pind = ProgressIndicator(totalToDo=os.path.getsize(fh.name), messagePrefix="completed", messageSuffix="of processing " + fh.name) except AttributeError: sys.stderr.write("WigIterator -- warning: " + "unable to show progress for stream") verbose = False chromsSeen = set() prev = None # NUMBERS = set(['1','2','3','4','5','6','7','8','9','0','.']) currentChrom, at, step = None, None, None for line in fh: line = line.strip() if line == "": continue if line[0] == 't' or line[0] == 'f': parts = line.split() if parts[0] == "track": continue elif parts[0] == "fixedStep": currentChrom = parts[1].split("=")[1] at = int(parts[2].split("=")[1]) step = int(parts[3].split("=")[1]) else: val = float(line) e = GenomicInterval(currentChrom, at, at + step, None, val, scoreType=scoreType) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise WigIteratorError("Wig file " + fd.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): raise WigIteratorError("Wig file " + fd.name + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e at += step if verbose : pind.done = fh.tell() pind.showProgress()
def __build_index(self, until=None, flush=False, verbose=False): """ build/expand the index for this file. :param until: expand the index until the record with this hash has been incorporated and then stop. If None, go until the iterator is exhausted. Note that if this hash is already in the index, no new items will be :param flush: if True, anything already in the index is discarded. """ assert (self._indexed_file_handle is not None) if flush: self._index = {} file_loc = self._indexed_file_handle.tell() if verbose: self._indexed_file_handle.seek(0, 2) # seek to end total = self._indexed_file_handle.tell() - file_loc self._indexed_file_handle.seek(file_loc) # back to where we were pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of building out index") for item in self.record_iterator(self._indexed_file_handle): hash_val = self.record_hash_function(item) self._index[hash_val] = file_loc file_loc = self._indexed_file_handle.tell() if until is not None and hash_val == until: break if verbose: pind.done = file_loc pind.showProgress()
def genericFileIterator(fn, verbose=False): """ @summary: iterate over a file, returning non-blank lines @param fn: either a string representing the name of the file or a file object @param verbose: if True, output status messages to stderr """ if type(fn).__name__ == "str": fh = open(fn) else: fh = fn if verbose: try: pind = ProgressIndicator(totalToDo=os.path.getsize(fh.name), messagePrefix="completed", messageSuffix="of processing " + fh.name) junk = fh.tell() except: sys.stderr.write( "Cannot show progress for stream.. doesn't behave " "like a file") verbose = False for line in fh: if verbose: pind.done = fh.tell() pind.showProgress() line = line.strip() if line == "": continue yield line
def __build_index(self, until=None, flush=False, verbose=False): """ build/expand the index for this file. :param until: expand the index until the record with this hash has been incorporated and then stop. If None, go until the iterator is exhausted. Note that if this hash is already in the index, no new items will be :param flush: if True, anything already in the index is discarded. """ assert(self._indexed_file_handle is not None) if flush: self._index = {} file_loc = self._indexed_file_handle.tell() if verbose: self._indexed_file_handle.seek(0, 2) # seek to end total = self._indexed_file_handle.tell() - file_loc self._indexed_file_handle.seek(file_loc) # back to where we were pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of building out index") for item in self.record_iterator(self._indexed_file_handle): hash_val = self.record_hash_function(item) self._index[hash_val] = file_loc file_loc = self._indexed_file_handle.tell() if until is not None and hash_val == until: break if verbose: pind.done = file_loc pind.showProgress()
def fastqIteratorComplex(fn, useMutableString=False, verbose=False): """ A generator function which yields FastqSequence objects read from a file or stream. This iterator can handle fastq files that have their sequence and/or their quality data split across multiple lines (i.e. there are newline characters in the sequence and quality strings). :param fn: A file-like stream or a string; if this is a string, it's treated as a filename specifying the location of an input fastq file, else it's treated as a file-like object, which must have a readline() method. :param useMustableString: if True, construct sequences from lists of chars, rather than python string objects, to allow more efficient editing. Use with caution. :param verbose: if True, print messages on progress to stderr. :param debug: if True, print debugging messages to stderr. :param sanger: if True, assume quality scores are in sanger format. Otherwise, assume they're in Illumina format. """ fh = fn if type(fh).__name__ == "str": fh = open(fh) prevLine = None # try to get an idea of how much data we have... pind = None if verbose: try: totalLines = linesInFile(fh.name) pind = ProgressIndicator( totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + fh.name ) except AttributeError: sys.stderr.write("fastqIterator -- warning: " + "unable to show progress for stream") verbose = False while True: # either we have a sequence header left over from the # prev call, or we need to read a new one from the file... # try to do that now name, prevLine = fastq_complex_parse_seq_header(fh, prevLine, pind, verbose) # read lines until we hit a qual header --> this is our sequence data seqdata, line = fastq_complex_parse_seq(fh, pind, verbose) # <line> is now a qual header, keep reading until we see # a sequence header.. or we run out of lines.. this is our quality data qualdata, line, prevLine = fastq_complex_parse_qual(fh, line, prevLine, verbose, pind) # package it all up.. yield NGSRead(seqdata, name, qualdata, useMutableString) if verbose: pind.showProgress() # remember where we stopped for next call, or finish prevLine = line if prevLine == "": break
def genericFileIterator(fn, verbose=False): """ @summary: iterate over a file, returning non-blank lines @param fn: either a string representing the name of the file or a file object @param verbose: if True, output status messages to stderr """ if type(fn).__name__ == "str": fh = open(fn) else: fh = fn if verbose: try: pind = ProgressIndicator(totalToDo=os.path.getsize(fh.name), messagePrefix="completed", messageSuffix="of processing " + fh.name) junk = fh.tell() except: sys.stderr.write("Cannot show progress for stream.. doesn't behave " "like a file") verbose = False for line in fh: if verbose: pind.done = fh.tell() pind.showProgress() line = line.strip() if line == "": continue yield line
def regularWigIterator(fd, verbose=False, sortedby=None, scoreType=int): """ @param sortedBy: if not None, should be one of ITERATOR_SORTED_BY_START indicating an order that the input stream must be sorted in @raise WigIteratorError: if sortedBy is set and stream is not sorted """ if verbose: try: totalLines = linesInFile(fd) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + getFDName(fd)) except AttributeError: sys.stderr.write("WigIterator -- warning: " + "unable to show progress for stream") verbose = False chromsSeen = set() prev = None fh = openFD(fd) for line in fh: if verbose: pind.done += 1 pind.showProgress() line = line.strip() if line == "": continue e = parseWigString(line, scoreType=scoreType) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise WigIteratorError( "bed file " + fd.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): raise WigIteratorError("BED file " + fd.name + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e
def repeat_masker_iterator(fh, alignment_index=None, header=True, verbose=False): """ Iterator for repeatmasker coordinate annotation files. These files describe the location of repeat occurrences. There is (optionally) a two-line header with the names of the fields (ignored by the iterator, if present). Each line is a record of an occurrence. The description of fields for each line is given in from_repeat_masker_string. :param fh: stream-like object, or string filename, to load the annotations from :param alignment_index: an IndexedFile for full alignments; keys should be repeat-masker IDs :param header: if True, expect and discard the two-line header; otherwise we will expect there is no header :param verbose: if True, output additional status messages about progress to stderr. """ strm = fh if type(fh).__name__ == "str": strm = open(fh) # try to get an idea of how much data we have... if verbose: try: total = os.path.getsize(strm.name) pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of processing " + strm.name) except AttributeError as e: sys.stderr.write(str(e)) sys.stderr.write("completed [unknown] of processing index") verbose = False if header: # chomp first 2 lines next(strm) next(strm) for line in strm: if verbose: pind.done = strm.tell() pind.showProgress() line = line.strip() if line == "": continue rto = retrotransposon.from_repeat_masker_string(line) if alignment_index is not None: rto.pairwise_alignment =\ JustInTimePairwiseAlignment(alignment_index, rto.uniq_id) yield rto
def regularWigIterator(fd, verbose=False, sortedby=None, scoreType=int): """ @param sortedBy: if not None, should be one of ITERATOR_SORTED_BY_START indicating an order that the input stream must be sorted in @raise WigIteratorError: if sortedBy is set and stream is not sorted """ if verbose: try: totalLines = linesInFile(fd) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + getFDName(fd)) except AttributeError: sys.stderr.write("WigIterator -- warning: " + "unable to show progress for stream") verbose = False chromsSeen = set() prev = None fh = openFD(fd) for line in fh: if verbose: pind.done += 1 pind.showProgress() line = line.strip() if line == "": continue e = parseWigString(line, scoreType=scoreType) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise WigIteratorError("bed file " + fd.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): raise WigIteratorError("BED file " + fd.name + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e
def __build_progress_indicator(fh): try: total = os.path.getsize(fh.name) except (AttributeError, OSError): raise ProgressIndicatorError("Failed to get max size for stream") pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of processing " + fh.name) return pind
def maskRegions(self, regions, verbose=False): """ Mask the given regions in this sequence with Ns. :param region: iterable of regions to mask. Each region can be any object with .start and .end attributes. Co-ords are zero based and inclusive of both end points. Any other attributes (e.g. chrom.) are ignored. :param verbose: print status messages to stderr if True """ if verbose: pind = ProgressIndicator(totalToDo=len(regions), messagePrefix="completed", messageSuffix="of masking regions in " + self.name) for region in regions: self.maskRegion(region) if verbose: pind.done += 1 pind.showProgress()
def intervalTrees(reffh, scoreType=int, verbose=False): """ Build a dictionary of interval trees indexed by chrom from a BED stream or file :param reffh: This can be either a string, or a stream-like object. In the former case, it is treated as a filename. The format of the file/stream must be BED. :param scoreType: The data type for scores (the fifth column) in the BED file. :param verbose: output progress messages to sys.stderr if True """ if type(reffh).__name__ == "str": fh = open(reffh) else: fh = reffh # load all the regions and split them into lists for each chrom elements = {} if verbose and fh != sys.stdin: totalLines = linesInFile(fh.name) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of loading " + fh.name) for element in BEDIterator(fh, scoreType=scoreType, verbose=verbose): if element.chrom not in elements: elements[element.chrom] = [] elements[element.chrom].append(element) if verbose and fh != sys.stdin: pind.done += 1 pind.showProgress() # create an interval tree for each list trees = {} if verbose: totalLines = len(elements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of making interval trees") for chrom in elements: trees[chrom] = IntervalTree(elements[chrom], openEnded=True) if verbose: pind.done += 1 pind.showProgress() return trees
def build_genome_alignment_from_file(ga_path, ref_spec, idx_path=None, verbose=False): """ build a genome alignment by loading from a single MAF file. :param ga_path: the path to the file to load. :param ref_spec: which species in the MAF file is the reference? :param idx_path: if provided, use this index to generate a just-in-time genome alignment, instead of loading the file immediately. """ blocks = [] if (idx_path is not None): bound_iter = functools.partial(genome_alignment_iterator, reference_species=ref_spec) hash_func = JustInTimeGenomeAlignmentBlock.build_hash factory = IndexedFile(None, bound_iter, hash_func) factory.read_index(idx_path, ga_path, verbose=verbose) pind = None for k in factory: if verbose: if pind is None: total = len(factory) pind = ProgressIndicator( totalToDo=total, messagePrefix="completed", messageSuffix="building alignment blocks ") pind.done += 1 pind.showProgress() blocks.append(JustInTimeGenomeAlignmentBlock(factory, k)) else: for b in genome_alignment_iterator(ga_path, ref_spec, verbose=verbose): blocks.append(b) return GenomeAlignment(blocks, verbose)
def intervalTreesFromList(inElements, verbose=False, openEnded=False): """ build a dictionary, indexed by chrom name, of interval trees for each chrom. :param inElements: list of genomic intervals. Members of the list must have chrom, start and end fields; no other restrictions. :param verbose: output progress messages to sys.stderr if True """ elements = {} if verbose: totalLines = len(inElements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of parsing") for element in inElements: if element.chrom not in elements: elements[element.chrom] = [] elements[element.chrom].append(element) if verbose: pind.done += 1 pind.showProgress() # create an interval tree for each list trees = {} if verbose: totalLines = len(elements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of making interval trees") for chrom in elements: trees[chrom] = IntervalTree(elements[chrom], openEnded) if verbose: pind.done += 1 pind.showProgress() return trees
def intervalTrees(reffh, scoreType=int, verbose=False): """ Build a dictionary of interval trees indexed by chrom from a BED stream or file :param reffh: This can be either a string, or a stream-like object. In the former case, it is treated as a filename. The format of the file/stream must be BED. :param scoreType: The data type for scores (the fifth column) in the BED file. :param verbose: output progress messages to sys.stderr if True """ if type(reffh).__name__ == "str": fh = open(reffh) else: fh = reffh # load all the regions and split them into lists for each chrom elements = {} if verbose and fh != sys.stdin: totalLines = linesInFile(fh.name) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of loading " + fh.name) for element in BEDIterator(fh, scoreType=scoreType, verbose=verbose): if element.chrom not in elements: elements[element.chrom] = [] elements[element.chrom].append(element) if verbose and fh != sys.stdin: pind.done += 1 pind.showProgress() # create an interval tree for each list trees = {} if verbose: totalLines = len(elements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of making interval trees") for chrom in elements: trees[chrom] = IntervalTree(elements[chrom], openEnded=True) if verbose: pind.done += 1 pind.showProgress() return trees
def intervalTreesFromList(inElements, verbose=False, openEnded=False): """ build a dictionary, indexed by chrom name, of interval trees for each chrom. :param inElements: list of genomic intervals. Members of the list must have chrom, start and end fields; no other restrictions. :param verbose: output progress messages to sys.stderr if True """ elements = {} if verbose: totalLines = len(inElements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of parsing") for element in inElements: if element.chrom not in elements: elements[element.chrom] = [] elements[element.chrom].append(element) if verbose: pind.done += 1 pind.showProgress() # create an interval tree for each list trees = {} if verbose: totalLines = len(elements) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of making interval trees") for chrom in elements: trees[chrom] = IntervalTree(elements[chrom], openEnded) if verbose: pind.done += 1 pind.showProgress() return trees
def fixedWigIterator(fd, verbose=False, sortedby=None, scoreType=int): """ @summary: """ fh = openFD(fd) if verbose: try: pind = ProgressIndicator(totalToDo=os.path.getsize(fh.name), messagePrefix="completed", messageSuffix="of processing " + fh.name) except AttributeError: sys.stderr.write("WigIterator -- warning: " + "unable to show progress for stream") verbose = False chromsSeen = set() prev = None # NUMBERS = set(['1','2','3','4','5','6','7','8','9','0','.']) currentChrom, at, step = None, None, None for line in fh: line = line.strip() if line == "": continue if line[0] == 't' or line[0] == 'f': parts = line.split() if parts[0] == "track": continue elif parts[0] == "fixedStep": currentChrom = parts[1].split("=")[1] at = int(parts[2].split("=")[1]) step = int(parts[3].split("=")[1]) else: val = float(line) e = GenomicInterval(currentChrom, at, at + step, None, val, scoreType=scoreType) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise WigIteratorError( "Wig file " + fd.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): raise WigIteratorError("Wig file " + fd.name + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e at += step if verbose: pind.done = fh.tell() pind.showProgress()
def fastqIteratorSimple(fn, verbose=False, allowNameMissmatch=False): """ A generator function that yields FastqSequence objects read from a fastq-format stream or filename. This is iterator requires that all sequence and quality data is provided on a single line -- put another way, it cannot parse fastq files with newline characters interspersed in the sequence and/or quality strings. That's probably okay though, as fastq files tend not to be formated like that (famous last words..). :param fn: filename or stream to read data from. :param allowNameMismatch: don't throw error if name in sequence data and quality data parts of a read don't match. Newer version of CASVA seem to output data like this, probably to save space. :param verbose: if True, output additional status messages to stderr about progress. """ fh = fn if type(fh).__name__ == "str": fh = open(fh) # try to get an idea of how much data we have... if verbose: try: totalLines = os.path.getsize(fh.name) pind = ProgressIndicator( totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + fh.name ) except AttributeError: sys.stderr.write("fastqIterator -- warning: " + "unable to show progress for stream") verbose = False while True: # read four lines.. if we can't get four lines, something is wrong lines = [] gotLines = 0 while gotLines < 4: l = fh.readline() if verbose: pind.done = fh.tell() pind.showProgress() if l == "": # end of file found... if gotLines == 0: # ok, not in the middle of a sequence break else: raise FastqFileFormatError("reached end of file in the " + "middle of sequence data") l = l.strip() if l == "": continue lines.append(l) gotLines += 1 # couldn't get any more data.. we're done if gotLines == 0: break # got our 4 lines, assemble our read.. # first check that names match if lines[0][1:] != lines[2][1:] and not allowNameMissmatch: raise FastqFileFormatError( "names in sequence don't match : " + str(lines[0][1:]) + " != " + str(lines[2][1:]) ) name = lines[0][1:] seq = lines[1] qual = lines[3] yield NGSRead(seq, name, qual)
def build(self): currentBlock = None at = self.handle.tell() seenChroms = set() lastIndexSeen = -1 if self.verbose: try: pind = ProgressIndicator( totalToDo=os.path.getsize(self.handle.name), messagePrefix="completed", messageSuffix="of building index for " + self.handle.name) except: sys.stderr.write("IndexedWig -- warning: " + "unable to show progress for stream\n") self.verbose = False ### note, for loop seems to buffer the file and so tell() gives a ### location that is not where the current line was read from, so ### we stick to readline instead. rline = None while rline != "": # get the next element rline = self.handle.readline() line = rline.strip() if line == "": continue e = parseWigString(line) # keep track of what chroms we've seen for checking order if not e.chrom in seenChroms: seenChroms.add(e.chrom) lastIndexSeen = -1 # check chrom order is ok for seenChrom in seenChroms: if seenChrom > e.chrom: msg = "wig file is not sorted, entry for chrom " + str(seenChrom) +\ " appears after entry for " + str(e.chrom) raise IndexedWigError(msg) # check position order is ok if e.start < lastIndexSeen: msg = "wig file is not sorted, entry for chrom " + str(e.chrom) +\ " at " + str(e.start) + " appears after " + str(lastIndexSeen) raise IndexedWigError(msg) # update the last index we've seen lastIndexSeen = e.end # debugging message if the current block is full if self.debug is True: sys.stderr.write("processing " + str(e)) if currentBlock is not None: sys.stderr.write("; is current block full?" + str(currentBlock.isfull()) + "\n") else: sys.stderr.write("\n") # we might need to make a new block for this element if currentBlock is None or currentBlock.isfull() or \ currentBlock.chrom != e.chrom: if self.debug: sys.stderr.write("making new block with " + str(e) + "\n") if currentBlock is not None: if self.debug: sys.stderr.write("closed block: " + str(currentBlock) + "\n") if currentBlock.chrom not in self.blocksByChrom: self.blocksByChrom[currentBlock.chrom] = [] self.blocksByChrom[currentBlock.chrom].append(currentBlock) currentBlock = WigBlock(at, e, self.blocksize) # add the element to the current block currentBlock.add(e) at = self.handle.tell() if self.verbose: pind.done = self.handle.tell() pind.showProgress() # don't forget to add the final block if currentBlock != None: if self.debug: sys.stderr.write("closed block: " + str(currentBlock) + "\n") if currentBlock.chrom not in self.blocksByChrom: self.blocksByChrom[currentBlock.chrom] = [] self.blocksByChrom[currentBlock.chrom].append(currentBlock) # build the interval trees for chrom in self.blocksByChrom: self.itrees[chrom] = IntervalTree(self.blocksByChrom[chrom], openEnded=True)
def build(self): currentBlock = None at = self.handle.tell() seenChroms = set() lastIndexSeen = -1 if self.verbose: try: pind = ProgressIndicator( totalToDo=os.path.getsize(self.handle.name), messagePrefix="completed", messageSuffix="of building index for " + self.handle.name, ) except: sys.stderr.write("IndexedWig -- warning: " + "unable to show progress for stream\n") self.verbose = False ### note, for loop seems to buffer the file and so tell() gives a ### location that is not where the current line was read from, so ### we stick to readline instead. rline = None while rline != "": # get the next element rline = self.handle.readline() line = rline.strip() if line == "": continue e = parseWigString(line) # keep track of what chroms we've seen for checking order if not e.chrom in seenChroms: seenChroms.add(e.chrom) lastIndexSeen = -1 # check chrom order is ok for seenChrom in seenChroms: if seenChrom > e.chrom: msg = ( "wig file is not sorted, entry for chrom " + str(seenChrom) + " appears after entry for " + str(e.chrom) ) raise IndexedWigError(msg) # check position order is ok if e.start < lastIndexSeen: msg = ( "wig file is not sorted, entry for chrom " + str(e.chrom) + " at " + str(e.start) + " appears after " + str(lastIndexSeen) ) raise IndexedWigError(msg) # update the last index we've seen lastIndexSeen = e.end # debugging message if the current block is full if self.debug is True: sys.stderr.write("processing " + str(e)) if currentBlock is not None: sys.stderr.write("; is current block full?" + str(currentBlock.isfull()) + "\n") else: sys.stderr.write("\n") # we might need to make a new block for this element if currentBlock is None or currentBlock.isfull() or currentBlock.chrom != e.chrom: if self.debug: sys.stderr.write("making new block with " + str(e) + "\n") if currentBlock is not None: if self.debug: sys.stderr.write("closed block: " + str(currentBlock) + "\n") if currentBlock.chrom not in self.blocksByChrom: self.blocksByChrom[currentBlock.chrom] = [] self.blocksByChrom[currentBlock.chrom].append(currentBlock) currentBlock = WigBlock(at, e, self.blocksize) # add the element to the current block currentBlock.add(e) at = self.handle.tell() if self.verbose: pind.done = self.handle.tell() pind.showProgress() # don't forget to add the final block if currentBlock != None: if self.debug: sys.stderr.write("closed block: " + str(currentBlock) + "\n") if currentBlock.chrom not in self.blocksByChrom: self.blocksByChrom[currentBlock.chrom] = [] self.blocksByChrom[currentBlock.chrom].append(currentBlock) # build the interval trees for chrom in self.blocksByChrom: self.itrees[chrom] = IntervalTree(self.blocksByChrom[chrom], openEnded=True)
def read_index(self, fh, indexed_fh, rec_iterator=None, rec_hash_func=None, parse_hash=str, flush=True, no_reindex=True, verbose=False): """ Populate this index from a file. Input format is just a tab-separated file, one record per line. The last column is the file location for the record and all columns before that are collectively considered to be the hash key for that record (which is probably only 1 column, but this allows us to permit tabs in hash keys). Lines consisting only of whitespace are skipped. :param fh: filename or stream-like object to read from. :param indexed_fh: either the filename of the indexed file or handle to it. :param rec_iterator: a function that will return an interator for the indexed file type (not the iterator for the file itself). This function must take a single argument which is the name the file to iterate over, or a stream like object similar to a filestream. :param rec_hash_func: a function that accepts the record type produced by the iterator and produces a unique hash for each record. :param parse_hash: a function to convert the string representation of the hash into whatever type is needed. By default, we just leave these as strings. :param flush: remove everything currently in the index and discard any details about a file that is already fully/partially indexed by this object. This is the default behavior. If False, then data from <fh> is just added to the existing index data (potentially overwriting some of it) and the existing index can continue to be used as before. :param no_reindex: if True, after loading the index, a missing key will cause an exception, rather than trigger re-scanning the indexed file for the associated record. The only reason to set this to False would be if your index was incomplete. :param verbose: output status message to STDERR about progress reading the index (if possible). :raise IndexError: on malformed line in input file/stream """ # set the record iterator and hash functions, if they were given if rec_iterator is not None: self.record_iterator = rec_iterator if rec_hash_func is not None: self.record_hash_function = rec_hash_func # disable re-indexing? self._no_reindex = no_reindex # figure out what kind of index identifier we got: handle or filename? handle = fh try: handle = open(fh) except TypeError: # okay, not a filename, we'll try treating it as a stream to read from. pass # clear this index? if flush: self._index = {} self._indexed_file_handle = None self._indexed_file_name = None # replace the name/handle for the indexed file indexed_fn = None try: # try treating this as a filename self.indexed_file = (indexed_fh, None) indexed_fn = indexed_fh except TypeError: try: # try treating this as a file handle self.indexed_file = (None, indexed_fh) except TypeError: fn = " from " + str(fh) if indexed_fn is not None else "" raise IndexError("failed to read index" + fn + "; " "reason: expected indexed filename or stream-like " "object, got " + str(type(indexed_fh))) # try to get an idea of how much data we have... if verbose: try: total = os.path.getsize(handle.name) pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of loading " + handle.name) except AttributeError as e: sys.stderr.write(str(e)) sys.stderr.write("completed [unknown] of loading index") verbose = False # read the index file and populate this object for line in handle: line = line.rstrip() if verbose: pind.done = handle.tell() pind.showProgress() if line.isspace(): continue parts = line.split("\t") if len(parts) < 2: raise IndexError("failed to parse line: '" + line + "'") key = parse_hash("\t".join(parts[:-1])) value = parts[-1] self._index[key] = int(value)
def repeat_masker_alignment_iterator(fn, index_friendly=True, verbose=False): """ Iterator for repeat masker alignment files; yields multiple alignment objects. Iterate over a file/stream of full repeat alignments in the repeatmasker format. Briefly, this format is as follows: each record (alignment) begins with a header line (see _rm_parse_header_line documentation for details of header format), followed by the alignment itself (example below) and finally a set of key-value meta-data pairs. The actual alignment looks like this:: chr1 11 CCCTGGAGATTCTTATT--AGTGATTTGGGCT 41 ii v -- v i i v C MER5B#DNA/hAT 10 CCCCAGAGATTCTGATTTAATTGGTCTGGGGT 42 chr1 42 GACTG 47 v C MER5B#DNA/hAT 43 CACTG 48 The 'C' indicates that its the reverse complement of the consensus. The central string gives information about matches; "-" indicates an insertion/deletion, "i" a transition (G<->A, C<->T) and "v" a transversion (all other substitutions). :param fh: filename or stream-like object to read from. :param index_friendly: if True, we will ensure the file/stream position is before the start of the record when we yield it; this requires the ability to seek within the stream though, so if iterating over a stream wtihout that ability, you'll have to set this to false. Further, this will disable buffering for the file, to ensure file.tell() behaves correctly, so a performance hit will be incurred. :param verbose: if true, output progress messages to stderr. """ # step 1 -- build our iterator for the stream.. try: fh = open(fn) except (TypeError): fh = fn iterable = fh if index_friendly: iterable = iter(fh.readline, '') # build progress indicator, if we want one and we're able to if verbose: try: m_fn = ": " + fh.name except TypeError: m_fn = "" try: current = fh.tell() fh.seek(0, 2) total_progress = fh.tell() fh.seek(current) pind = ProgressIndicator(totalToDo=total_progress, messagePrefix="completed", messageSuffix="of processing repeat-masker " "alignment file" + m_fn) except IOError: pind = None old_fh_pos = None new_fh_pos = fh.tell() s1 = None s2 = None s1_name = None s2_name = None s1_start = None s1_end = None s2_start = None s2_end = None meta_data = None alignment_line_counter = 0 alig_l_space = 0 prev_seq_len = 0 rev_comp_match = None remaining_repeat = None remaining_genomic = None for line in iterable: if verbose and pind is not None: pind.done = fh.tell() pind.showProgress() if index_friendly: old_fh_pos = new_fh_pos new_fh_pos = fh.tell() line = line.rstrip() if line.lstrip() == "" and alignment_line_counter % 3 != 1: continue s_pres_split = re.split(r'(\s+)', line) parts = [x for x in s_pres_split if not (x.isspace() or x == "")] n = len(parts) for i in REPEATMASKER_FIELDS_TO_TRIM: if n >= i + 1: parts[i] = parts[i].strip() # decide what to do with this line -- is it a header line, part of the # alignment or a meta-data key-value line if alignment_line_counter % 3 == 1: if (REPEATMASKER_VALIDATE_MUTATIONS and not _rm_is_valid_annotation_line(line)): raise IOError("invalid mutation line: " + line) l_space = _rm_compute_leading_space(s_pres_split) - alig_l_space pad_right = prev_seq_len - (l_space + len(line.strip())) meta_data[ANNOTATION_KEY] += ((' ' * l_space) + line.strip() + (' ' * pad_right)) alignment_line_counter += 1 elif _rm_is_header_line(parts, n): if not (s1 is None and s2 is None and meta_data is None): if ANNOTATION_KEY in meta_data: meta_data[ANNOTATION_KEY] = meta_data[ANNOTATION_KEY].rstrip() if index_friendly: fh.seek(old_fh_pos) ss1 = Sequence(s1_name, s1, s1_start, s1_end, "+", remaining_genomic) s2s = "-" if rev_comp_match else "+" ss2 = Sequence(s2_name, s2, s2_start, s2_end, s2s, remaining_repeat) yield PairwiseAlignment(ss1, ss2, meta_data) if index_friendly: fh.seek(new_fh_pos) meta_data = {} s1 = "" s2 = "" s1_name, s2_name = _rm_get_names_from_header(parts) s1_start, s1_end = _rm_get_reference_coords_from_header(parts) s2_start, s2_end = _rm_get_repeat_coords_from_header(parts) rev_comp_match = _rm_is_reverse_comp_match(parts) remaining_repeat = _rm_get_remaining_repeat_from_header(parts) remaining_genomic = _rm_get_remaining_genomic_from_header(parts) _rm_parse_header_line(parts, meta_data) alignment_line_counter = 0 elif _rm_is_alignment_line(parts, s1_name, s2_name): alignment_line_counter += 1 name, seq = _rm_extract_sequence_and_name(parts, s1_name, s2_name) if name == s1_name: s1 += seq elif name == s2_name: s2 += seq alig_l_space = _rm_compute_leading_space_alig(s_pres_split, seq) prev_seq_len = len(seq) else: k, v = _rm_parse_meta_line(parts) meta_data[k] = v if index_friendly: fh.seek(old_fh_pos) ss1 = Sequence(s1_name, s1, s1_start, s1_end, "+", remaining_genomic) s2s = "-" if rev_comp_match else "+" ss2 = Sequence(s2_name, s2, s2_start, s2_end, s2s, remaining_repeat) yield PairwiseAlignment(ss1, ss2, meta_data) if index_friendly: fh.seek(new_fh_pos)
def repeat_masker_alignment_iterator(fn, index_friendly=True, verbose=False): """ Iterator for repeat masker alignment files; yields multiple alignment objects. Iterate over a file/stream of full repeat alignments in the repeatmasker format. Briefly, this format is as follows: each record (alignment) begins with a header line (see _rm_parse_header_line documentation for details of header format), followed by the alignment itself (example below) and finally a set of key-value meta-data pairs. The actual alignment looks like this:: chr1 11 CCCTGGAGATTCTTATT--AGTGATTTGGGCT 41 ii v -- v i i v C MER5B#DNA/hAT 10 CCCCAGAGATTCTGATTTAATTGGTCTGGGGT 42 chr1 42 GACTG 47 v C MER5B#DNA/hAT 43 CACTG 48 The 'C' indicates that its the reverse complement of the consensus. The central string gives information about matches; "-" indicates an insertion/deletion, "i" a transition (G<->A, C<->T) and "v" a transversion (all other substitutions). :param fh: filename or stream-like object to read from. :param index_friendly: if True, we will ensure the file/stream position is before the start of the record when we yield it; this requires the ability to seek within the stream though, so if iterating over a stream wtihout that ability, you'll have to set this to false. Further, this will disable buffering for the file, to ensure file.tell() behaves correctly, so a performance hit will be incurred. :param verbose: if true, output progress messages to stderr. """ # step 1 -- build our iterator for the stream.. try: fh = open(fn) except (TypeError): fh = fn iterable = fh if index_friendly: iterable = iter(fh.readline, '') # build progress indicator, if we want one and we're able to if verbose: try: m_fn = ": " + fh.name except TypeError: m_fn = "" try: current = fh.tell() fh.seek(0, 2) total_progress = fh.tell() fh.seek(current) pind = ProgressIndicator( totalToDo=total_progress, messagePrefix="completed", messageSuffix="of processing repeat-masker " "alignment file" + m_fn) except IOError: pind = None old_fh_pos = None new_fh_pos = fh.tell() s1 = None s2 = None s1_name = None s2_name = None s1_start = None s1_end = None s2_start = None s2_end = None meta_data = None alignment_line_counter = 0 alig_l_space = 0 prev_seq_len = 0 rev_comp_match = None remaining_repeat = None remaining_genomic = None for line in iterable: if verbose and pind is not None: pind.done = fh.tell() pind.showProgress() if index_friendly: old_fh_pos = new_fh_pos new_fh_pos = fh.tell() line = line.rstrip() if line.lstrip() == "" and alignment_line_counter % 3 != 1: continue s_pres_split = re.split(r'(\s+)', line) parts = [x for x in s_pres_split if not (x.isspace() or x == "")] n = len(parts) for i in REPEATMASKER_FIELDS_TO_TRIM: if n >= i + 1: parts[i] = parts[i].strip() # decide what to do with this line -- is it a header line, part of the # alignment or a meta-data key-value line if alignment_line_counter % 3 == 1: if (REPEATMASKER_VALIDATE_MUTATIONS and not _rm_is_valid_annotation_line(line)): raise IOError("invalid mutation line: " + line) l_space = _rm_compute_leading_space(s_pres_split) - alig_l_space pad_right = prev_seq_len - (l_space + len(line.strip())) meta_data[ANNOTATION_KEY] += ((' ' * l_space) + line.strip() + (' ' * pad_right)) alignment_line_counter += 1 elif _rm_is_header_line(parts, n): if not (s1 is None and s2 is None and meta_data is None): if ANNOTATION_KEY in meta_data: meta_data[ANNOTATION_KEY] = meta_data[ ANNOTATION_KEY].rstrip() if index_friendly: fh.seek(old_fh_pos) ss1 = Sequence(s1_name, s1, s1_start, s1_end, "+", remaining_genomic) s2s = "-" if rev_comp_match else "+" ss2 = Sequence(s2_name, s2, s2_start, s2_end, s2s, remaining_repeat) yield PairwiseAlignment(ss1, ss2, meta_data) if index_friendly: fh.seek(new_fh_pos) meta_data = {} s1 = "" s2 = "" s1_name, s2_name = _rm_get_names_from_header(parts) s1_start, s1_end = _rm_get_reference_coords_from_header(parts) s2_start, s2_end = _rm_get_repeat_coords_from_header(parts) rev_comp_match = _rm_is_reverse_comp_match(parts) remaining_repeat = _rm_get_remaining_repeat_from_header(parts) remaining_genomic = _rm_get_remaining_genomic_from_header(parts) _rm_parse_header_line(parts, meta_data) alignment_line_counter = 0 elif _rm_is_alignment_line(parts, s1_name, s2_name): alignment_line_counter += 1 name, seq = _rm_extract_sequence_and_name(parts, s1_name, s2_name) if name == s1_name: s1 += seq elif name == s2_name: s2 += seq alig_l_space = _rm_compute_leading_space_alig(s_pres_split, seq) prev_seq_len = len(seq) else: k, v = _rm_parse_meta_line(parts) meta_data[k] = v if index_friendly: fh.seek(old_fh_pos) ss1 = Sequence(s1_name, s1, s1_start, s1_end, "+", remaining_genomic) s2s = "-" if rev_comp_match else "+" ss2 = Sequence(s2_name, s2, s2_start, s2_end, s2s, remaining_repeat) yield PairwiseAlignment(ss1, ss2, meta_data) if index_friendly: fh.seek(new_fh_pos)
def fastqIteratorComplex(fn, useMutableString=False, verbose=False): """ A generator function which yields FastqSequence objects read from a file or stream. This iterator can handle fastq files that have their sequence and/or their quality data split across multiple lines (i.e. there are newline characters in the sequence and quality strings). :param fn: A file-like stream or a string; if this is a string, it's treated as a filename specifying the location of an input fastq file, else it's treated as a file-like object, which must have a readline() method. :param useMustableString: if True, construct sequences from lists of chars, rather than python string objects, to allow more efficient editing. Use with caution. :param verbose: if True, print messages on progress to stderr. :param debug: if True, print debugging messages to stderr. :param sanger: if True, assume quality scores are in sanger format. Otherwise, assume they're in Illumina format. """ fh = fn if type(fh).__name__ == "str": fh = open(fh) prevLine = None # try to get an idea of how much data we have... pind = None if verbose: try: totalLines = linesInFile(fh.name) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + fh.name) except AttributeError: sys.stderr.write("fastqIterator -- warning: " + "unable to show progress for stream") verbose = False while True: # either we have a sequence header left over from the # prev call, or we need to read a new one from the file... # try to do that now name, prevLine = fastq_complex_parse_seq_header(fh, prevLine, pind, verbose) # read lines until we hit a qual header --> this is our sequence data seqdata, line = fastq_complex_parse_seq(fh, pind, verbose) # <line> is now a qual header, keep reading until we see # a sequence header.. or we run out of lines.. this is our quality data qualdata, line, prevLine = fastq_complex_parse_qual(fh, line, prevLine, verbose, pind) # package it all up.. yield NGSRead(seqdata, name, qualdata, useMutableString) if verbose: pind.showProgress() # remember where we stopped for next call, or finish prevLine = line if prevLine == "": break
def fastqIteratorSimple(fn, verbose=False, allowNameMissmatch=False): """ A generator function that yields FastqSequence objects read from a fastq-format stream or filename. This is iterator requires that all sequence and quality data is provided on a single line -- put another way, it cannot parse fastq files with newline characters interspersed in the sequence and/or quality strings. That's probably okay though, as fastq files tend not to be formated like that (famous last words..). :param fn: filename or stream to read data from. :param allowNameMismatch: don't throw error if name in sequence data and quality data parts of a read don't match. Newer version of CASVA seem to output data like this, probably to save space. :param verbose: if True, output additional status messages to stderr about progress. """ fh = fn if type(fh).__name__ == "str": fh = open(fh) # try to get an idea of how much data we have... if verbose: try: totalLines = os.path.getsize(fh.name) pind = ProgressIndicator(totalToDo=totalLines, messagePrefix="completed", messageSuffix="of processing " + fh.name) except AttributeError: sys.stderr.write("fastqIterator -- warning: " + "unable to show progress for stream") verbose = False while True: # read four lines.. if we can't get four lines, something is wrong lines = [] gotLines = 0 while gotLines < 4: l = fh.readline() if verbose: pind.done = fh.tell() pind.showProgress() if l == "": # end of file found... if gotLines == 0: # ok, not in the middle of a sequence break else: raise FastqFileFormatError("reached end of file in the " + "middle of sequence data") l = l.strip() if l == "": continue lines.append(l) gotLines += 1 # couldn't get any more data.. we're done if gotLines == 0: break # got our 4 lines, assemble our read.. # first check that names match if lines[0][1:] != lines[2][1:] and not allowNameMissmatch: raise FastqFileFormatError("names in sequence don't match : " + str(lines[0][1:]) + " != " + str(lines[2][1:])) name = lines[0][1:] seq = lines[1] qual = lines[3] yield NGSRead(seq, name, qual)
def read_index(self, fh, indexed_fh, rec_iterator=None, rec_hash_func=None, parse_hash=str, flush=True, no_reindex=True, verbose=False): """ Populate this index from a file. Input format is just a tab-separated file, one record per line. The last column is the file location for the record and all columns before that are collectively considered to be the hash key for that record (which is probably only 1 column, but this allows us to permit tabs in hash keys). Lines consisting only of whitespace are skipped. :param fh: filename or stream-like object to read from. :param indexed_fh: either the filename of the indexed file or handle to it. :param rec_iterator: a function that will return an interator for the indexed file type (not the iterator for the file itself). This function must take a single argument which is the name the file to iterate over, or a stream like object similar to a filestream. :param rec_hash_func: a function that accepts the record type produced by the iterator and produces a unique hash for each record. :param parse_hash: a function to convert the string representation of the hash into whatever type is needed. By default, we just leave these as strings. :param flush: remove everything currently in the index and discard any details about a file that is already fully/partially indexed by this object. This is the default behavior. If False, then data from <fh> is just added to the existing index data (potentially overwriting some of it) and the existing index can continue to be used as before. :param no_reindex: if True, after loading the index, a missing key will cause an exception, rather than trigger re-scanning the indexed file for the associated record. The only reason to set this to False would be if your index was incomplete. :param verbose: output status message to STDERR about progress reading the index (if possible). :raise IndexError: on malformed line in input file/stream """ # set the record iterator and hash functions, if they were given if rec_iterator is not None: self.record_iterator = rec_iterator if rec_hash_func is not None: self.record_hash_function = rec_hash_func # disable re-indexing? self._no_reindex = no_reindex # figure out what kind of index identifier we got: handle or filename? handle = fh try: handle = open(fh) except TypeError: # okay, not a filename, we'll try treating it as a stream to read from. pass # clear this index? if flush: self._index = {} self._indexed_file_handle = None self._indexed_file_name = None # replace the name/handle for the indexed file indexed_fn = None try: # try treating this as a filename self.indexed_file = (indexed_fh, None) indexed_fn = indexed_fh except TypeError: try: # try treating this as a file handle self.indexed_file = (None, indexed_fh) except TypeError: fn = " from " + str(fh) if indexed_fn is not None else "" raise IndexError( "failed to read index" + fn + "; " "reason: expected indexed filename or stream-like " "object, got " + str(type(indexed_fh))) # try to get an idea of how much data we have... if verbose: try: total = os.path.getsize(handle.name) pind = ProgressIndicator(totalToDo=total, messagePrefix="completed", messageSuffix="of loading " + handle.name) except AttributeError as e: sys.stderr.write(str(e)) sys.stderr.write("completed [unknown] of loading index") verbose = False # read the index file and populate this object for line in handle: line = line.rstrip() if verbose: pind.done = handle.tell() pind.showProgress() if line.isspace(): continue parts = line.split("\t") if len(parts) < 2: raise IndexError("failed to parse line: '" + line + "'") key = parse_hash("\t".join(parts[:-1])) value = parts[-1] self._index[key] = int(value)
def BEDIterator(filehandle, sortedby=None, verbose=False, scoreType=int, dropAfter=None): """ Get an iterator for a BED file :param filehandle: this can be either a string, or a stream-like object. In the former case, it is treated as a filename. The format of the file/stream must be BED. :param sortedby: if None, order is not checked. if == ITERATOR_SORTED_START, elements in file must be sorted by chrom and start index (an exception is raised if they are not) if == ITERATOR_SORTED_END, element must be sorted by chrom and end index. :param verbose: if True, output additional progress messages to stderr :param scoreType: The data type for scores (the fifth column) in the BED file. :param dropAfter: an int indicating that any fields after and including this field should be ignored as they don't conform to the BED format. By default, None, meaning we use all fields. Index from zero. :return: iterator where subsequent calls to next() yield the next BED element in the stream as a GenomicInterval object. """ chromsSeen = set() prev = None if type(filehandle).__name__ == "str": filehandle = open(filehandle) if verbose: try: pind = ProgressIndicator(totalToDo=os.path.getsize(filehandle.name), messagePrefix="completed", messageSuffix="of processing " + filehandle.name) except (AttributeError, OSError) as e: sys.stderr.write("BEDIterator -- warning: " + "unable to show progress for stream") verbose = False for line in filehandle: if verbose: pind.done = filehandle.tell() pind.showProgress() if line.strip() == "": continue try: e = parseBEDString(line, scoreType, dropAfter=dropAfter) except GenomicIntervalError as e: raise BEDError(str(e) + " on line " + line) # sorting by name? if ((sortedby == ITERATOR_SORTED_NAME and prev is not None) and (prev.name > e.name)): raise BEDError("bed file " + filehandle.name + " not sorted by element name" + " found " + e.name + " after " + prev.name) # first item if prev is None: chromsSeen.add(e.chrom) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise BEDError("bed file " + filehandle.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) if sortedby == ITERATOR_SORTED_END and prev.end > e.end: raise BEDError("bed file " + filehandle.name + " not sorted by end index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START or sortedby == ITERATOR_SORTED_END or sortedby == ITERATOR_SORTED_CHROM) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): try: e_fn = filehandle.name except AttributeError: e_fn = "UNNAMED STREAM" raise BEDError("BED file " + e_fn + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e
def BEDIterator(filehandle, sortedby=None, verbose=False, scoreType=int, dropAfter=None): """ Get an iterator for a BED file :param filehandle: this can be either a string, or a stream-like object. In the former case, it is treated as a filename. The format of the file/stream must be BED. :param sortedby: if None, order is not checked. if == ITERATOR_SORTED_START, elements in file must be sorted by chrom and start index (an exception is raised if they are not) if == ITERATOR_SORTED_END, element must be sorted by chrom and end index. :param verbose: if True, output additional progress messages to stderr :param scoreType: The data type for scores (the fifth column) in the BED file. :param dropAfter: an int indicating that any fields after and including this field should be ignored as they don't conform to the BED format. By default, None, meaning we use all fields. Index from zero. :return: iterator where subsequent calls to next() yield the next BED element in the stream as a GenomicInterval object. """ chromsSeen = set() prev = None if type(filehandle).__name__ == "str": filehandle = open(filehandle) if verbose: try: pind = ProgressIndicator( totalToDo=os.path.getsize(filehandle.name), messagePrefix="completed", messageSuffix="of processing " + filehandle.name) except (AttributeError, OSError) as e: sys.stderr.write("BEDIterator -- warning: " + "unable to show progress for stream") verbose = False for line in filehandle: if verbose: pind.done = filehandle.tell() pind.showProgress() if line.strip() == "": continue try: e = parseBEDString(line, scoreType, dropAfter=dropAfter) except GenomicIntervalError as e: raise BEDError(str(e) + " on line " + line) # sorting by name? if ((sortedby == ITERATOR_SORTED_NAME and prev is not None) and (prev.name > e.name)): raise BEDError("bed file " + filehandle.name + " not sorted by element name" + " found " + e.name + " after " + prev.name) # first item if prev is None: chromsSeen.add(e.chrom) # on same chrom as the prev item, make sure order is right if prev is not None and sortedby is not None and e.chrom == prev.chrom: if sortedby == ITERATOR_SORTED_START and prev.start > e.start: raise BEDError("bed file " + filehandle.name + " not sorted by start index - saw item " + str(prev) + " before " + str(e)) if sortedby == ITERATOR_SORTED_END and prev.end > e.end: raise BEDError("bed file " + filehandle.name + " not sorted by end index - saw item " + str(prev) + " before " + str(e)) # starting a new chrom.. make sure we haven't already seen it if prev is not None and prev.chrom != e.chrom: if (sortedby == ITERATOR_SORTED_START or sortedby == ITERATOR_SORTED_END or sortedby == ITERATOR_SORTED_CHROM) and\ (e.chrom in chromsSeen or prev.chrom > e.chrom): try: e_fn = filehandle.name except AttributeError: e_fn = "UNNAMED STREAM" raise BEDError("BED file " + e_fn + " not sorted by chrom") chromsSeen.add(e.chrom) # all good.. yield e prev = e