def _processPrimers(self, primer_fn_forward, primer_fn_reverse, window_size, primer_out_fn,
revcmp_primers=False):
"""
Do basic sanity checks that:
(1) all primers in forward start with f_xxx and are unique
(2) all primers in reverse start with r_xxx and are unique
(3) check that no forward primers appear in reverse primers (no symmetry)
(4) write the primers (f_xxx, f_xxx_revcmp, r_xxx, r_xxx_revcmp) all to one primer file
"""
def sanity_check_primers(reader, prefix):
"""
Go through the primers, check that the prefix exists and all seqs are unique
"""
primers = {} # primer -> sequence, but can also contain the revcmp version with _revcmp suffix
for r in reader:
if not r.name.startswith(prefix):
errMsg = "Forward primer should start with f_, but saw:", r.name
raise ClassifierException(errMsg)
if len(r.sequence) > window_size:
errMsg = "Primer {n} has length {l} which is longer than {k}.".\
format(n=r.name, l=len(r.sequence), k=window_size)
logging.error(errMsg)
raise ClassifierException(errMsg)
ss = r.sequence.upper()
if ss in primers.itervalues():
errMsg = "Duplicate sequences found for", ss
raise ClassifierException(errMsg)
primers[r.name.strip()] = r.sequence
# revcmp not needed becuz phmmer does both strands apparently...
#primers[r.name.strip() + "_revcmp"] = revcmp(r.sequence)
return primers
logging.info("Process primers for {case}.".
format(case=("finding primers" if not revcmp_primers
else "detecting chimeras")))
reader_f = FastaReader(primer_fn_forward)
reader_r = FastaReader(primer_fn_reverse)
primers_f = sanity_check_primers(reader_f, prefix="f_")
primers_r = sanity_check_primers(reader_r, prefix="r_")
reader_f.close()
reader_r.close()
same_seqs = set(primers_f.values()).intersection(primers_r.values())
if len(same_seqs) > 0:
errMsg = "Identical sequences found in both Forward/Reverse!\n"
errMsg += "\n".join(same_seqs)
raise ClassifierException(errMsg)
# Write Fi and reverse-complemented Ri to primer_out_fn
with open(primer_out_fn, 'w') as f:
for (name, seq) in primers_f.iteritems():
f.write(">{n}\n{s}\n".format(n=name, s=seq))
for (name, seq) in primers_r.iteritems():
f.write(">{n}\n{s}\n".format(n=name, s=revcmp(seq)))
return primers_f.keys() + primers_r.keys()
def _chunkReads(self,
reads_fn,
reads_per_chunk,
chunked_reads_fns,
extract_front_back_only=True,
window_size=100):
"""Split reads within reads_fn into multiple chunks each containing
at most 'reads_per_chunk' reads, save to files in 'chunked_reads_fns'.
If extract_front_back_only is true, extract the first and the last
'window_size' bases and save them as readname_front and readname_back.
Otherwise, copy read names and sequences entirely.
"""
logging.debug("Split {f} into ".format(f=reads_fn) +
"{n} chunks, ".format(n=len(chunked_reads_fns)) +
"each containing at most {n} reads.".format(
n=reads_per_chunk))
if extract_front_back_only:
logging.debug("Extract exactly {k} bases from front" +
" and end of each read.".format(k=window_size))
freader = FastaReader(reads_fn)
chunkIndex = -1
fwriter = None
for i, read in enumerate(freader):
if i % reads_per_chunk == 0:
chunkIndex += 1
if fwriter is not None:
fwriter.close()
fwriter = None
fwriter = open(chunked_reads_fns[chunkIndex], 'w')
rcseq = revcmp(read.sequence)
if extract_front_back_only:
fwriter.write(">{n}_front\n{s}\n>{n}_back\n{rcs}\n".format(
n=read.name,
s=read.sequence[:window_size],
rcs=rcseq[:window_size]))
else:
fwriter.write(">{n}\n{s}\n".format(n=read.name,
s=read.sequence))
if fwriter is not None:
fwriter.close()
def _chunkReads(self, reads_fn, reads_per_chunk, chunked_reads_fns,
extract_front_back_only=True, window_size=100):
"""Split reads within reads_fn into multiple chunks each containing
at most 'reads_per_chunk' reads, save to files in 'chunked_reads_fns'.
If extract_front_back_only is true, extract the first and the last
'window_size' bases and save them as readname_front and readname_back.
Otherwise, copy read names and sequences entirely.
"""
logging.debug("Split {f} into ".format(f=reads_fn) +
"{n} chunks, ".format(n=len(chunked_reads_fns)) +
"each containing at most {n} reads.".
format(n=reads_per_chunk))
if extract_front_back_only:
logging.debug("Extract exactly {k} bases from front" +
" and end of each read.".format(k=window_size))
freader = FastaReader(reads_fn)
chunkIndex = -1
fwriter = None
for i, read in enumerate(freader):
if i % reads_per_chunk == 0:
chunkIndex += 1
if fwriter is not None:
fwriter.close()
fwriter = None
fwriter = open(chunked_reads_fns[chunkIndex], 'w')
rcseq = revcmp(read.sequence)
if extract_front_back_only:
fwriter.write(">{n}_front\n{s}\n>{n}_back\n{rcs}\n".format(
n=read.name, s=read.sequence[:window_size],
rcs=rcseq[:window_size]))
else:
fwriter.write(">{n}\n{s}\n".format(n=read.name,
s=read.sequence))
if fwriter is not None:
fwriter.close()
def _trimBarCode(self, reads_fn, out_fl_reads_fn, out_nfl_reads_fn,
primer_report_nfl_fn,
best_of_front, best_of_back, primer_indices,
min_seq_len, min_score, change_read_id,
ignore_polyA):
"""Trim bar code from reads in 'reads_fn', annotate each read,
indicating:
whether its 5' primer, 3' primer and polyA tail are seen,
start positions of its 5' primer, 3' primer and polyA tail,
and primer info.
, save non-full-length reads to 'out_nfl_reads_fn',
, save full-length reads to 'out_fl_reads_fn', which can later be
used in chimera detection
, write primer info of nfl reads to _primer_report_nfl_fn.
Note that chimera detection is not necessary for nfl reads, but
is required for fl reads. So we only write primer info for nfl here
and will write primer info for fl reads when chimera detection
is done.
best_of_front/Back: {read_id: {primer_name:DOMRecord}}
min_seq_len: minimum length to output a read.
min_score: minimum score to output a read.
change_read_id: if True, change read ids to 'movie/zmw/start_end'.
"""
logging.info("Trim bar code away from reads.")
logging.debug("Writing full-length trimmed reads to {f}".
format(f=out_fl_reads_fn))
logging.debug("Writing non-full-length trimmed reads to {f}".
format(f=out_nfl_reads_fn))
logging.debug("Writing primer reports before chimera detection to {f}".
format(f=primer_report_nfl_fn))
with FastaReader(reads_fn) as fareader, \
FastaWriter(out_nfl_reads_fn) as nfl_fawriter, \
FastaWriter(out_fl_reads_fn) as fl_fawriter, \
open(primer_report_nfl_fn, 'w') as reporter:
for read in fareader:
self.summary.num_reads += 1 # number of ROI reads
pbread = PBRead(read)
logging.debug("Pick up best primer combo for {r}".
format(r=read.name))
primerIndex, strand, fw, rc = self._pickBestPrimerCombo(
best_of_front[read.name], best_of_back[read.name],
primer_indices, min_score)
logging.debug("read={0}\n".format(read.name) +
"primer={0} strand={1} fw={2} rc={3}".
format(primerIndex, strand, fw, rc))
if fw is None and rc is None:
# No primer seen in this sequence, classified
# as non-full-length
newName = pbread.name
if change_read_id:
newName = "{m}/{z}/{s1}_{e1}{isccs}".format(
m=pbread.movie, z=pbread.zmw,
s1=pbread.start, e1=pbread.end,
isccs=("_CCS" if pbread.isCCS else ""))
annotation = ReadAnnotation(ID=newName)
# Write reports of nfl reads
reporter.write(annotation.toReportRecord(delimitor=",") + "\n")
if len(read.sequence) >= min_seq_len:
# output non-full-length reads to nfl.trimmed.fasta
nfl_fawriter.writeRecord(annotation.toAnnotation(),
read.sequence)
self.summary.num_nfl += 1
else:
self.summary.num_filtered_short_reads += 1
continue
seq = read.sequence if strand == "+" else revcmp(read.sequence)
five_end, three_start = None, None
if fw is not None:
five_end = fw.sEnd
self.summary.num_5_seen += 1
if rc is not None:
three_start = len(seq) - rc.sEnd
self.summary.num_3_seen += 1
s, e = pbread.start, pbread.end
# Try to find polyA tail in read
polyAPos = self._findPolyA(seq, three_start=three_start)
if polyAPos >= 0: # polyA found
seq = seq[:polyAPos]
e1 = s + polyAPos if strand == "+" else e - polyAPos
self.summary.num_polyA_seen += 1
elif three_start is not None: # polyA not found
seq = seq[:three_start]
e1 = s + three_start if strand == "+" else e - three_start
else:
e1 = e if strand == "+" else s
if five_end is not None:
seq = seq[five_end:]
s1 = s + five_end if strand == "+" else e - five_end
else:
s1 = s if strand == "+" else e
newName = pbread.name
if change_read_id:
newName = "{m}/{z}/{s1}_{e1}{isccs}".format(
m=pbread.movie, z=pbread.zmw, s1=s1, e1=e1,
isccs=("_CCS" if pbread.isCCS else ""))
# Create an annotation
annotation = ReadAnnotation(ID=newName, strand=strand,
fiveend=five_end, polyAend=polyAPos,
threeend=three_start, primer=primerIndex,
ignore_polyA=ignore_polyA)
# Write reports for nfl reads
if annotation.isFullLength is not True:
reporter.write(annotation.toReportRecord(delimitor=",") + "\n")
if len(seq) >= min_seq_len:
if annotation.isFullLength is True:
# Write long full-length reads
fl_fawriter.writeRecord(annotation.toAnnotation(), seq)
self.summary.num_fl += 1
else:
# Write long non-full-length reads.
nfl_fawriter.writeRecord(annotation.toAnnotation(), seq)
self.summary.num_nfl += 1
else:
self.summary.num_filtered_short_reads += 1
raise ClassifierException(errMsg)
if len(r.sequence) > window_size:
errMsg = "Primer {n} has length {l} which is longer than {k}.".\
format(n=expectedName, l=len(r.sequence), k=window_size)
logging.error(errMsg)
raise ClassifierException(errMsg)
if direction == "F":
# Save >Fi and Fi_sequence.
primers.append([expectedName, r.sequence])
else: # direction is "R"
# fwdF/fwdR is the forward sequence of Fi/Ri
fwdF, fwdR = primers[-1][1], r.sequence
# revcmpF/revcmpR is the reverse complement of Fi/Ri
revcmpF, revcmpR = revcmp(fwdF), revcmp(fwdR)
# If Fi and Ri are reverse complementariliy identical, bail out,
# because we need Poly A tail to distinguish Fi and Ri.
if fwdF.find(revcmpR) >= 0 or revcmpR.find(fwdF) >= 0:
infoMsg = "Primer F{n}, R{n} ".format(n=primerComboId) + \
"are reverse complementarily identical. " + \
"Need to add 'AAAA' to 3' to distinguish them."
logging.info(infoMsg)
if revcmp_primers is False:
# Save primer Ri and revcmp(Ri_sequence) + TTTT
primers.append([expectedName, revcmpR + "T" * 4])
else: # revcmp_primers is True
primers.append([expectedName, "A" * 4 + fwdR])
primers.append(['F{n}_revcmp'.format(n=primerComboId),
revcmpF])
primers.append(['R{n}_revcmp'.format(n=primerComboId),
def _trimBarCode(self, reads_fn, out_fl_reads_fn, out_nfl_reads_fn,
primer_report_nfl_fn,
best_of_front, best_of_back, primer_names,
min_seq_len, min_score, change_read_id,
ignore_polyA, keep_primer):
"""Trim bar code from reads in 'reads_fn', annotate each read,
indicating:
whether its 5' primer, 3' primer and polyA tail are seen,
start positions of its 5' primer, 3' primer and polyA tail,
and primer info.
, save non-full-length reads to 'out_nfl_reads_fn',
, save full-length reads to 'out_fl_reads_fn', which can later be
used in chimera detection
, write primer info of nfl reads to _primer_report_nfl_fn.
Note that chimera detection is not necessary for nfl reads, but
is required for fl reads. So we only write primer info for nfl here
and will write primer info for fl reads when chimera detection
is done.
best_of_front/Back: {read_id: {primer_name:DOMRecord}}
min_seq_len: minimum length to output a read.
min_score: minimum score to output a read.
change_read_id: if True, change read ids to 'movie/zmw/start_end'.
"""
logging.info("Trim bar code away from reads.")
logging.debug("Writing full-length trimmed reads to {f}".
format(f=out_fl_reads_fn))
logging.debug("Writing non-full-length trimmed reads to {f}".
format(f=out_nfl_reads_fn))
logging.debug("Writing primer reports before chimera detection to {f}".
format(f=primer_report_nfl_fn))
with FastaReader(reads_fn) as fareader, \
FastaWriter(out_nfl_reads_fn) as nfl_fawriter, \
FastaWriter(out_fl_reads_fn) as fl_fawriter, \
open(primer_report_nfl_fn, 'w') as reporter:
for read in fareader:
self.summary.num_reads += 1 # number of ROI reads
pbread = PBRead(read)
logging.debug("Pick up best primer combo for {r}".
format(r=read.name))
primerName, strand, fw, rc = self._pickBestPrimerCombo(
best_of_front[read.name], best_of_back[read.name],
primer_names, min_score)
logging.debug("read={0}\n".format(read.name) +
"strand={0} fw={1} rc={2}".
format(strand, fw, rc))
if (strand == '?') or (fw is None and rc is None):
# No primer seen in this sequence, classified
# as non-full-length
newName = pbread.name
if change_read_id:
newName = "{m}/{z}/{s1}_{e1}{isccs}".format(
m=pbread.movie, z=pbread.zmw,
s1=pbread.start, e1=pbread.end,
isccs=("_CCS" if pbread.isCCS else ""))
annotation = ReadAnnotation(ID=newName, primer=primerName)
# Write reports of nfl reads
reporter.write(annotation.toReportRecord(delimitor=",") + "\n")
if len(read.sequence) >= min_seq_len:
# output non-full-length reads to nfl.trimmed.fasta
nfl_fawriter.writeRecord(annotation.toAnnotation(),
read.sequence)
self.summary.num_nfl += 1
else:
self.summary.num_filtered_short_reads += 1
continue
seq = read.sequence if strand == "+" else revcmp(read.sequence)
five_end, three_start = None, None
if fw is not None:
five_end = fw.sEnd
self.summary.num_5_seen += 1
if rc is not None:
three_start = len(seq) - rc.sEnd
self.summary.num_3_seen += 1
s, e = pbread.start, pbread.end
# Try to find polyA tail in read
polyAPos = self._findPolyA(seq, three_start=three_start)
if polyAPos >= 0 and not ignore_polyA: # polyA found and not to ignore it
if not keep_primer:
seq = seq[:polyAPos]
e1 = s + polyAPos if strand == "+" else e - polyAPos
else:
e1 = e if strand == '+' else s
self.summary.num_polyA_seen += 1
elif three_start is not None: # polyA not found but 3' found
if not keep_primer:
seq = seq[:three_start]
e1 = s + three_start if strand == "+" else e - three_start
else:
e1 = e if strand == '+' else s
else: # polyA not found and 3' not found
e1 = e if strand == "+" else s
if five_end is not None:
if not keep_primer:
seq = seq[five_end:]
s1 = s + five_end if strand == "+" else e - five_end
else:
s1 = s if strand == '+' else e
else:
s1 = s if strand == "+" else e
newName = pbread.name
if change_read_id:
newName = "{m}/{z}/{s1}_{e1}{isccs}".format(
m=pbread.movie, z=pbread.zmw, s1=s1, e1=e1,
isccs=("_CCS" if pbread.isCCS else ""))
# Create an annotation
annotation = ReadAnnotation(ID=newName, strand=strand,
fiveend=five_end, polyAend=polyAPos,
threeend=three_start, primer=primerName,
ignore_polyA=ignore_polyA)
# Write reports for nfl reads
if annotation.isFullLength is not True:
reporter.write(annotation.toReportRecord(delimitor=",") + "\n")
if len(seq) >= min_seq_len:
if annotation.isFullLength is True:
# Write long full-length reads
fl_fawriter.writeRecord(annotation.toAnnotation(), seq)
self.summary.num_fl += 1
else:
# Write long non-full-length reads.
nfl_fawriter.writeRecord(annotation.toAnnotation(), seq)
self.summary.num_nfl += 1
else:
self.summary.num_filtered_short_reads += 1
reader_f.close()
reader_r.close()
same_seqs = set(primers_f.values()).intersection(primers_r.values())
if len(same_seqs) > 0:
errMsg = "Identical sequences found in both Forward/Reverse!\n"
errMsg += "\n".join(same_seqs)
raise ClassifierException(errMsg)
# Write Fi and reverse-complemented Ri to primer_out_fn
with open(primer_out_fn, 'w') as f:
for (name, seq) in primers_f.iteritems():
f.write(">{n}\n{s}\n".format(n=name, s=seq))
for (name, seq) in primers_r.iteritems():
f.write(">{n}\n{s}\n".format(n=name, s=revcmp(seq)))
return primers_f.keys() + primers_r.keys()
@property
def numReads(self):
"""Return the number of reads in reads_fn."""
cmd = "grep -c '>' {r}".format(r=real_upath(self.reads_fn))
output, errCode, errMsg = backticks(cmd)
if errCode != 0:
raise ClassifierException("Error reading file {r}:{e}".format(
r=self.reads_fn, e=str(errMsg)))
return int(output[0])
def _chunkReads(self,
reads_fn,
reads_per_chunk,
def _processPrimers(self, primer_fn, window_size, primer_out_fn,
revcmp_primers=False):
"""
Check and generate primers.
1. Check primers in primer_fn are in order F0, R0, F1, R1, ...
Fn, Rn, and lengths are all < k, where k is the primer search
window length.
F0 5' NNNNNNNNNN 3'
R0 3' NNNNNNNNNN 5'
2. If Ri and Fi are revers complementarily identical,
add a polyA tail to 3' of Ri.
3. For each combo of primers Fi and Ri, save the following to
primer_out_fn.
3.1 If revcmp_primers is False,
>Fi
Fi_sequence
>Ri
revcmp(Ri_sequence)
3.2 If revcmp_primers is True,
>Fi
Fi_sequence
>Ri
Ri_sequence
>Fi_revcmp
revcmp(Fi_sqeuence)
>Ri_revcmp
revcmp(Ri_sqeuence)
4. return primers range(0, n)
"""
logging.info("Process primers for {case}.".
format(case=("finding primers" if not revcmp_primers
else "detecting chimeras")))
freader = FastaReader(primer_fn)
primers = []
primerComboId = -1
for i, r in enumerate(freader):
if i % 2 == 0:
direction = "F"
primerComboId += 1
else:
direction = "R"
expectedName = "{d}{n}".format(d=direction, n=primerComboId)
if r.name != expectedName:
errMsg = "Primers should be placed in order F0, R0, F1, R1..."
logging.error(errMsg)
raise ClassifierException(errMsg)
if len(r.sequence) > window_size:
errMsg = "Primer {n} has length {l} which is longer than {k}.".\
format(n=expectedName, l=len(r.sequence), k=window_size)
logging.error(errMsg)
raise ClassifierException(errMsg)
if direction == "F":
# Save >Fi and Fi_sequence.
primers.append([expectedName, r.sequence])
else: # direction is "R"
# fwdF/fwdR is the forward sequence of Fi/Ri
fwdF, fwdR = primers[-1][1], r.sequence
# revcmpF/revcmpR is the reverse complement of Fi/Ri
revcmpF, revcmpR = revcmp(fwdF), revcmp(fwdR)
# If Fi and Ri are reverse complementariliy identical, bail out,
# because we need Poly A tail to distinguish Fi and Ri.
if fwdF.find(revcmpR) >= 0 or revcmpR.find(fwdF) >= 0:
infoMsg = "Primer F{n}, R{n} ".format(n=primerComboId) + \
"are reverse complementarily identical. " + \
"Need to add 'AAAA' to 3' to distinguish them."
logging.info(infoMsg)
if revcmp_primers is False:
# Save primer Ri and revcmp(Ri_sequence) + TTTT
primers.append([expectedName, revcmpR + "T" * 4])
else: # revcmp_primers is True
primers.append([expectedName, "A" * 4 + fwdR])
primers.append(['F{n}_revcmp'.format(n=primerComboId),
revcmpF])
primers.append(['R{n}_revcmp'.format(n=primerComboId),
revcmpR + "T" * 4])
else: # Ri and Fi are not revcmp identical
if revcmp_primers is False:
# Save >Ri and revcmp(Ri_sequence)
primers.append([expectedName, revcmpR])
else:
# Save >Ri and Ri_sequence
primers.append([expectedName, fwdR])
# Save >Fi_revcmp and revcmp(Fi_sequence)
primers.append(['F{n}_revcmp'.format(n=primerComboId),
revcmpF])
# Save >Ri_revcmp and revcmp(Ri_sequence)
primers.append(['R{n}_revcmp'.format(n=primerComboId),
revcmpR])
freader.close()
# Write Fi and reverse-complemented Ri to primer_out_fn
f = open(primer_out_fn, 'w')
for (name, seq) in primers:
f.write(">{n}\n{s}\n".format(n=name, s=seq))
f.close()
return range(0, primerComboId + 1)
