def build_fwtrack (self, opt): """Build FWTrackII from all lines, return a FWTrackII object. Note: All locations will be merged (exclude the same location) then sorted after the track is built. If both_strand is True, it will store strand information in FWTrackII object. if do_merge is False, it will not merge the same location after the track is built. """ fwtrack = FWTrackII() i = 0 m = 0 for thisline in self.fhd: (chromosome,fpos,strand) = self._fw_parse_line(thisline) i+=1 if i == 1000000: m += 1 logging.info(" %d" % (m*1000000)) i=0 if not fpos or not chromosome: continue fwtrack.add_loc(chromosome,fpos,strand) return fwtrack
def build_fwtrack (self, opt, dist=200): """Build FWTrackII from all lines, return a FWTrackII object. lfhd: the filehandler for left tag file rfhd: the filehandler for right tag file dist: the best distance between two tags in a pair The score system for pairing two tags: score = abs(abs(rtag-ltag)-200)+error4lefttag+error4righttag the smaller score the better pairing. If the score for a pairing is bigger than 200, this pairing will be discarded. Note only the best pair is kept. If there are over two best pairings, this pair of left and right tags will be discarded. Note, the orders in left tag file and right tag file must match, i.e., the Nth left tag must has the same name as the Nth right tag. Note, remove comment lines beforehand. """ fwtrack = FWTrackII() i = 0 m = 0 lnext = self.lfhd.next rnext = self.rfhd.next self.dist = dist try: while 1: lline = lnext() rline = rnext() (chromname,fpos,strand) = self._fw_parse_line(lline,rline) i+=1 if i == 1000000: m += 1 logging.info(" %d" % (m*1000000)) i=0 if not fpos or not chromname: continue try: chromname = chromname[:chromname.rindex(".fa")] except ValueError: pass fwtrack.add_loc(chromname,fpos,strand) except StopIteration: pass return fwtrack
def build_fwtrack (self, opt): """Build FWTrackII from all lines, return a FWTrackII object. """ fwtrack = FWTrackII() i = 0 m = 0 for thisline in self.fhd: (chromosome,fpos,strand) = self._fw_parse_line(thisline) i+=1 if i == 1000000: m += 1 logging.info(" %d" % (m*1000000)) i=0 if not fpos or not chromosome: continue fwtrack.add_loc(chromosome,fpos,strand) return fwtrack
def build_fwtrack(self, opt): """Build FWTrackII from all lines, return a FWTrackII object. Note only the unique match for a tag is kept. """ fwtrack = FWTrackII() i = 0 m = 0 references = [] fseek = self.fhd.seek fread = self.fhd.read ftell = self.fhd.tell #move to pos 4, there starts something fseek(4) header_len = struct.unpack('<i', fread(4))[0] fseek(header_len + ftell()) #get the number of chromosome nc = struct.unpack('<i', fread(4))[0] for x in range(nc): # read each chromosome name nlength = struct.unpack('<i', fread(4))[0] references.append(fread(nlength)[:-1]) # jump over chromosome size, we don't need it fseek(ftell() + 4) while 1: try: entrylength = struct.unpack('<i', fread(4))[0] except struct.error: break (chrid, fpos, strand) = self._fw_binary_parse(fread(entrylength)) i += 1 if i == 1000000: m += 1 logging.info(" %d" % (m * 1000000)) i = 0 if fpos >= 0: fwtrack.add_loc(references[chrid], fpos, strand) self.fhd.close() return fwtrack
def build_fwtrack (self, opt): """Build FWTrackII from all lines, return a FWTrackII object. Note only the unique match for a tag is kept. """ fwtrack = FWTrackII() i = 0 m = 0 references = [] fseek = self.fhd.seek fread = self.fhd.read ftell = self.fhd.tell #move to pos 4, there starts something fseek(4) header_len = struct.unpack('<i', fread(4))[0] fseek(header_len + ftell()) #get the number of chromosome nc = struct.unpack('<i', fread(4))[0] for x in range(nc): # read each chromosome name nlength = struct.unpack('<i', fread(4))[0] references.append(fread(nlength)[:-1]) # jump over chromosome size, we don't need it fseek(ftell() + 4) while 1: try: entrylength = struct.unpack('<i', fread(4))[0] except struct.error: break (chrid,fpos,strand) = self._fw_binary_parse(fread(entrylength)) i+=1 if i == 1000000: m += 1 logging.info(" %d" % (m*1000000)) i=0 if fpos >= 0: fwtrack.add_loc(references[chrid],fpos,strand) self.fhd.close() return fwtrack
def build_fwtrack(self, opt): """Build FWTrackII from all lines, return a FWTrackII object. Handle multi-reads here by building a probability and enrichment index or select only one alignment from each multi-read. Initial alignment probabilities are set from read/mismatch qualities or from a uniform distribution. """ fwtrack = FWTrackII() i = 0 m = 0 read_total = 0 recent_tags = [] random_select_one_multi = opt.random_select_one_multi no_multi_reads = opt.no_multi_reads min_score = opt.min_score prior_prob_snp = opt.prior_prob_snp no_prior_prob_map = opt.no_prior_prob_map if opt.qual_scale == 'auto': opt.qual_scale = self._guess_qual_scale() if opt.qual_scale == 'sanger+33': qual_offset = 33 elif opt.qual_scale == 'illumina+64': qual_offset = 64 group_starts_append = fwtrack.group_starts.append fwtrack_add_loc = fwtrack.add_loc match_probs = { } # {(1,30):p(match|phred=30), (0,30):p(mismatch|phred=30)} for grouplines in self._group_by_name(self.fhd): read_total += 1 # in ratios, only count reads, not total alignments if len(grouplines) == 1: # uniquely mapping reads i += 1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m * 1000000)) i = 0 chromosome, fpos, strand, qualstr, mismatches = grouplines[0] fwtrack_add_loc(chromosome, fpos, strand, 0) # 0'th index => unique else: if no_multi_reads: # throw away multi-reads fwtrack.total -= 1 continue elif random_select_one_multi: # choose one alignment at random i += 1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m * 1000000)) i = 0 randline = grouplines[random_range(len(grouplines))] chromosome, fpos, strand, qualstr, mismatches = randline fwtrack_add_loc(chromosome, fpos, strand, 0) else: # use all alignments probabilistically group_starts_append( fwtrack.total_multi + 1) # starts at 1 (0 reserved for unique reads) if no_prior_prob_map: # don't use map quality; just assume uniform priors for (chromosome, fpos, strand, qualstr, mismatches) in grouplines: i += 1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m * 1000000)) i = 0 fwtrack.total_multi += 1 fwtrack_add_loc(chromosome, fpos, strand, fwtrack.total_multi) normed_probs = [1. / len(grouplines)] * len(grouplines) else: # TODO: might want to be working in log space-- if many mismatches, we'll lose precision qualstr = grouplines[0][ 3] # all quality strings are shared across the group group_total_prob = 0. group_probs = [] group_probs_append = group_probs.append for (chromosome, fpos, strand, qualstr, mismatches) in grouplines: i += 1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m * 1000000)) i = 0 fwtrack.total_multi += 1 fwtrack_add_loc(chromosome, fpos, strand, fwtrack.total_multi) mismatches = set(mismatches) read_prob = 1. # P(SNP) = prior probability a SNP occurs at any base # P(SE) = probability there was a sequencing error (from PHRED) # _P(Map|SNP,SE)__MATCH__SNP__SE_ # 0 0 0 0 # can't map here without explanation # 1 0 0 1 # 1 0 1 0 # 1 0 1 1 # 1 1 0 0 # 1 1 0 1 # 0 1 1 0 # wouldn't map here if SNP, but sequencer read reference # 1 1 1 1 # we are interested in P(Mapping | Match), which is equivalent to: # \Sum_{SNP \in {0,1}, SE \in {0,1}} p(SNP) * p(SE) * p(Map|SE,SNP), or: # p(Map|match = 0): # p(SE) + p(SNP) + p(SE)*p(SNP) # p(Map|match = 1): # 1 - (p(SE) + p(SE)*p(SNP)) for b in xrange(len(qualstr)): tup = (b in mismatches, qualstr[b]) if tup in match_probs: prob = match_probs[tup] elif tup[0]: # mismatch p_seq_error = 10.**( (qualstr[b] - qual_offset) / -10.) prob = p_seq_error + prior_prob_snp + p_seq_error * prior_prob_snp match_probs[tup] = prob else: # match p_seq_error = 10.**( (qualstr[b] - qual_offset) / -10.) prob = 1. - (p_seq_error + p_seq_error * prior_prob_snp) match_probs[tup] = prob read_prob *= prob # quick & dirty check-- only looking at last base assert qualstr[ b] >= qual_offset # Specified quality scale yielded a negative phred score! You probably have the wrong PHRED scale! assert 0. <= read_prob <= 1. # error with map qualities #raise BaseQualityError("Specified quality scale yielded a negative phred score! You probably have the wrong PHRED scale!") group_probs_append(read_prob) group_total_prob += read_prob normed_probs = [ p / group_total_prob for p in group_probs ] fwtrack.prob_aligns.extend(normed_probs) fwtrack.prior_aligns.extend(normed_probs) fwtrack.enrich_scores.extend([min_score] * len(grouplines)) fwtrack.total = read_total # overwrite the running total, counting each read once return fwtrack
def build_fwtrack (self, opt): """Build FWTrackII from all lines, return a FWTrackII object. Handle multi-reads here by building a probability and enrichment index or select only one alignment from each multi-read. Initial alignment probabilities are set from read/mismatch qualities or from a uniform distribution. """ fwtrack = FWTrackII() i = 0 m = 0 read_total = 0 recent_tags = [] random_select_one_multi = opt.random_select_one_multi no_multi_reads = opt.no_multi_reads min_score = opt.min_score prior_prob_snp = opt.prior_prob_snp no_prior_prob_map = opt.no_prior_prob_map if opt.qual_scale == 'auto': opt.qual_scale = self._guess_qual_scale() if opt.qual_scale == 'sanger+33': qual_offset = 33 elif opt.qual_scale == 'illumina+64': qual_offset = 64 group_starts_append = fwtrack.group_starts.append fwtrack_add_loc = fwtrack.add_loc match_probs = {} # {(1,30):p(match|phred=30), (0,30):p(mismatch|phred=30)} for grouplines in self._group_by_name(self.fhd): read_total += 1 # in ratios, only count reads, not total alignments if len(grouplines) == 1: # uniquely mapping reads i+=1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m*1000000)) i=0 chromosome, fpos, strand, qualstr, mismatches = grouplines[0] fwtrack_add_loc(chromosome,fpos,strand,0) # 0'th index => unique else: if no_multi_reads: # throw away multi-reads fwtrack.total -= 1 continue elif random_select_one_multi: # choose one alignment at random i+=1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m*1000000)) i=0 randline = grouplines[random_range(len(grouplines))] chromosome,fpos,strand,qualstr,mismatches = randline fwtrack_add_loc(chromosome,fpos,strand,0) else: # use all alignments probabilistically group_starts_append(fwtrack.total_multi + 1) # starts at 1 (0 reserved for unique reads) if no_prior_prob_map: # don't use map quality; just assume uniform priors for (chromosome,fpos,strand, qualstr,mismatches) in grouplines: i+=1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m*1000000)) i=0 fwtrack.total_multi += 1 fwtrack_add_loc(chromosome,fpos,strand, fwtrack.total_multi) normed_probs = [1./len(grouplines)] * len(grouplines) else: # TODO: might want to be working in log space-- if many mismatches, we'll lose precision qualstr = grouplines[0][3] # all quality strings are shared across the group group_total_prob = 0. group_probs = [] group_probs_append = group_probs.append for (chromosome,fpos,strand, qualstr, mismatches) in grouplines: i+=1 if i == 1000000: m += 1 logging.info(" %d alignments read." % (m*1000000)) i=0 fwtrack.total_multi += 1 fwtrack_add_loc(chromosome,fpos,strand, fwtrack.total_multi) mismatches = set(mismatches) read_prob = 1. # P(SNP) = prior probability a SNP occurs at any base # P(SE) = probability there was a sequencing error (from PHRED) # _P(Map|SNP,SE)__MATCH__SNP__SE_ # 0 0 0 0 # can't map here without explanation # 1 0 0 1 # 1 0 1 0 # 1 0 1 1 # 1 1 0 0 # 1 1 0 1 # 0 1 1 0 # wouldn't map here if SNP, but sequencer read reference # 1 1 1 1 # we are interested in P(Mapping | Match), which is equivalent to: # \Sum_{SNP \in {0,1}, SE \in {0,1}} p(SNP) * p(SE) * p(Map|SE,SNP), or: # p(Map|match = 0): # p(SE) + p(SNP) + p(SE)*p(SNP) # p(Map|match = 1): # 1 - (p(SE) + p(SE)*p(SNP)) for b in xrange(len(qualstr)): tup = (b in mismatches,qualstr[b]) if tup in match_probs: prob = match_probs[tup] elif tup[0]: # mismatch p_seq_error = 10. ** ((qualstr[b]-qual_offset)/-10.) prob = p_seq_error + prior_prob_snp + p_seq_error * prior_prob_snp match_probs[tup] = prob else: # match p_seq_error = 10. ** ((qualstr[b]-qual_offset)/-10.) prob = 1. - (p_seq_error + p_seq_error * prior_prob_snp) match_probs[tup] = prob read_prob *= prob # quick & dirty check-- only looking at last base assert qualstr[b] >= qual_offset # Specified quality scale yielded a negative phred score! You probably have the wrong PHRED scale! assert 0.<=read_prob<=1. # error with map qualities #raise BaseQualityError("Specified quality scale yielded a negative phred score! You probably have the wrong PHRED scale!") group_probs_append(read_prob) group_total_prob += read_prob normed_probs = [p / group_total_prob for p in group_probs] fwtrack.prob_aligns.extend(normed_probs) fwtrack.prior_aligns.extend(normed_probs) fwtrack.enrich_scores.extend([min_score] * len(grouplines)) fwtrack.total = read_total # overwrite the running total, counting each read once return fwtrack