def _flushContigIfCompleted(self, window):
refId, _, _ = window
refEntry = reference.byName[refId]
refName = refEntry.fullName
basesProcessed = self.referenceBasesProcessedById[refId]
requiredBases = reference.numReferenceBases(refId,
options.referenceWindows)
if basesProcessed == requiredBases:
# This contig is done, so we can dump to file and delete
# the data structures.
if self.gffWriter or self.vcfWriter:
variants = sorted(self.variantsByRefId[refId])
if self.gffWriter:
self.gffWriter.writeVariants(variants)
if self.vcfWriter:
self.vcfWriter.writeVariants(variants)
del self.variantsByRefId[refId]
#
# If the user asked to analyze a window or a set of
# windows, we output a FAST[AQ] contig per analyzed
# window. Otherwise we output a fasta contig per
# reference contig.
#
# We try to be intelligent about naming the output
# contigs, to include window information where applicable.
#
for span in reference.enumerateSpans(refId,
options.referenceWindows):
_, s, e = span
if (s == 0) and (e == refEntry.length):
spanName = refName
else:
spanName = refName + "_%d_%d" % (s, e)
cssName = consensus.consensusContigName(
spanName, self._algorithmName)
# Gather just the chunks pertaining to this span
chunksThisSpan = [
chunk for chunk in self.consensusChunksByRefId[refId]
if windows.windowsIntersect(chunk.refWindow, span)
]
css = consensus.join(chunksThisSpan)
if self.fastaWriter:
self.fastaWriter.writeRecord(cssName, css.sequence)
if self.fastqWriter:
self.fastqWriter.writeRecord(cssName, css.sequence,
css.confidence)
del self.consensusChunksByRefId[refId]
def _flushContigIfCompleted(self, window):
refId, _, _ = window
refEntry = reference.byId[refId]
refName = refEntry.name
basesProcessed = self.referenceBasesProcessedById[refId]
requiredBases = reference.numReferenceBases(refId, options.referenceWindows)
if basesProcessed == requiredBases:
# This contig is done, so we can dump to file and delete
# the data structures.
if self.gffWriter:
self.gffWriter.writeVariants(sorted(self.variantsByRefId[refId]))
del self.variantsByRefId[refId]
#
# If the user asked to analyze a window or a set of
# windows, we output a FAST[AQ] contig per analyzed
# window. Otherwise we output a fasta contig per
# reference contig.
#
# We try to be intelligent about naming the output
# contigs, to include window information where applicable.
#
for span in reference.enumerateSpans(refId, options.referenceWindows):
_, s, e = span
if (s == 0) and (e == refEntry.length):
spanName = refName
else:
spanName = refName + ":%d-%d" % (s, e)
cssName = consensus.consensusContigName(spanName,
options.algorithm)
# Gather just the chunks pertaining to this span
chunksThisSpan = [ chunk for chunk in self.consensusChunksByRefId[refId]
if windows.windowsIntersect(chunk.refWindow, span) ]
css = consensus.join(chunksThisSpan)
if self.fastaWriter:
self.fastaWriter.writeRecord(cssName,
css.sequence)
if self.fastqWriter:
self.fastqWriter.writeRecord(cssName,
css.sequence,
css.confidence)
del self.consensusChunksByRefId[refId]
def consensusAndVariantsForWindow(alnFile, refWindow, referenceContig,
depthLimit, arrowConfig):
"""
High-level routine for calling the consensus for a
window of the genome given a cmp.h5.
Identifies the coverage contours of the window in order to
identify subintervals where a good consensus can be called.
Creates the desired "no evidence consensus" where there is
inadequate coverage.
"""
winId, winStart, winEnd = refWindow
logging.info("Arrow operating on %s" %
reference.windowToString(refWindow))
if options.fancyChunking:
# 1) identify the intervals with adequate coverage for arrow
# consensus; restrict to intervals of length > 10
alnHits = U.readsInWindow(alnFile, refWindow,
depthLimit=20000,
minMapQV=arrowConfig.minMapQV,
strategy="longest",
stratum=options.readStratum,
barcode=options.barcode)
starts = np.fromiter((hit.tStart for hit in alnHits), np.int)
ends = np.fromiter((hit.tEnd for hit in alnHits), np.int)
intervals = kSpannedIntervals(refWindow, arrowConfig.minPoaCoverage,
starts, ends, minLength=10)
coverageGaps = holes(refWindow, intervals)
allIntervals = sorted(intervals + coverageGaps)
if len(allIntervals) > 1:
logging.info("Usable coverage in %s: %r" %
(reference.windowToString(refWindow), intervals))
else:
allIntervals = [ (winStart, winEnd) ]
# 2) pull out the reads we will use for each interval
# 3) call consensusForAlignments on the interval
subConsensi = []
variants = []
for interval in allIntervals:
intStart, intEnd = interval
intRefSeq = referenceContig[intStart:intEnd]
subWin = subWindow(refWindow, interval)
windowRefSeq = referenceContig[intStart:intEnd]
alns = U.readsInWindow(alnFile, subWin,
depthLimit=depthLimit,
minMapQV=arrowConfig.minMapQV,
strategy="longest",
stratum=options.readStratum,
barcode=options.barcode)
clippedAlns_ = [ aln.clippedTo(*interval) for aln in alns ]
clippedAlns = U.filterAlns(subWin, clippedAlns_, arrowConfig)
if len([ a for a in clippedAlns
if a.spansReferenceRange(*interval) ]) >= arrowConfig.minPoaCoverage:
logging.debug("%s: Reads being used: %s" %
(reference.windowToString(subWin),
" ".join([str(hit.readName) for hit in alns])))
css = U.consensusForAlignments(subWin,
intRefSeq,
clippedAlns,
arrowConfig)
siteCoverage = U.coverageInWindow(subWin, alns)
variants_ = U.variantsFromConsensus(subWin, windowRefSeq,
css.sequence, css.confidence, siteCoverage,
options.aligner,
ai=None)
filteredVars = filterVariants(options.minCoverage,
options.minConfidence,
variants_)
# Annotate?
if options.annotateGFF:
annotateVariants(filteredVars, clippedAlns)
variants += filteredVars
# Dump?
shouldDumpEvidence = \
((options.dumpEvidence == "all") or
(options.dumpEvidence == "variants") and (len(variants) > 0))
if shouldDumpEvidence:
logging.info("Arrow does not yet support --dumpEvidence")
# dumpEvidence(options.evidenceDirectory,
# subWin, windowRefSeq,
# clippedAlns, css)
else:
css = ArrowConsensus.noCallConsensus(arrowConfig.noEvidenceConsensus,
subWin, intRefSeq)
subConsensi.append(css)
# 4) glue the subwindow consensus objects together to form the
# full window consensus
css = join(subConsensi)
# 5) Return
return css, variants
def consensusAndVariantsForWindow(alnFile, refWindow, referenceContig,
depthLimit, arrowConfig):
"""
High-level routine for calling the consensus for a
window of the genome given a BAM file.
Identifies the coverage contours of the window in order to
identify subintervals where a good consensus can be called.
Creates the desired "no evidence consensus" where there is
inadequate coverage.
"""
winId, winStart, winEnd = refWindow
logging.info("Arrow operating on %s" % reference.windowToString(refWindow))
if options.fancyChunking:
# 1) identify the intervals with adequate coverage for arrow
# consensus; restrict to intervals of length > 10
alnHits = U.readsInWindow(alnFile,
refWindow,
depthLimit=20000,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
starts = np.fromiter((hit.tStart for hit in alnHits), np.int)
ends = np.fromiter((hit.tEnd for hit in alnHits), np.int)
intervals = kSpannedIntervals(refWindow,
arrowConfig.minPoaCoverage,
starts,
ends,
minLength=10)
coverageGaps = holes(refWindow, intervals)
allIntervals = sorted(intervals + coverageGaps)
if len(allIntervals) > 1:
logging.info("Usable coverage in %s: %r" %
(reference.windowToString(refWindow), intervals))
else:
allIntervals = [(winStart, winEnd)]
# 2) pull out the reads we will use for each interval
# 3) call consensusForAlignments on the interval
subConsensi = []
variants = []
for interval in allIntervals:
intStart, intEnd = interval
intRefSeq = referenceContig[intStart:intEnd]
subWin = subWindow(refWindow, interval)
windowRefSeq = referenceContig[intStart:intEnd]
alns = U.readsInWindow(alnFile,
subWin,
depthLimit=depthLimit,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
clippedAlns_ = [aln.clippedTo(*interval) for aln in alns]
clippedAlns = U.filterAlns(subWin, clippedAlns_, arrowConfig)
if len([a for a in clippedAlns if a.spansReferenceRange(*interval)
]) >= arrowConfig.minPoaCoverage:
logging.debug("%s: Reads being used: %s" %
(reference.windowToString(subWin), " ".join(
[str(hit.readName) for hit in alns])))
alnsUsed = [] if options.reportEffectiveCoverage else None
css = U.consensusForAlignments(subWin,
intRefSeq,
clippedAlns,
arrowConfig,
alnsUsed=alnsUsed)
# Tabulate the coverage implied by these alignments, as
# well as the post-filtering ("effective") coverage
siteCoverage = U.coverageInWindow(subWin, alns)
effectiveSiteCoverage = U.coverageInWindow(
subWin, alnsUsed) if options.reportEffectiveCoverage else None
variants_, newPureCss = U.variantsFromConsensus(
subWin,
windowRefSeq,
css.sequence,
css.confidence,
siteCoverage,
effectiveSiteCoverage,
options.aligner,
ai=None,
diploid=arrowConfig.polishDiploid)
# Annotate?
if options.annotateGFF:
annotateVariants(variants_, clippedAlns)
variants += variants_
# The nascent consensus sequence might contain ambiguous bases, these
# need to be removed as software in the wild cannot deal with such
# characters and we only use IUPAC for *internal* bookkeeping.
if arrowConfig.polishDiploid:
css.sequence = newPureCss
else:
css = ArrowConsensus.noCallConsensus(
arrowConfig.noEvidenceConsensus, subWin, intRefSeq)
subConsensi.append(css)
# 4) glue the subwindow consensus objects together to form the
# full window consensus
css = join(subConsensi)
# 5) Return
return css, variants
def consensusAndVariantsForWindow(alnFile, refWindow, referenceContig,
depthLimit, arrowConfig):
"""
High-level routine for calling the consensus for a
window of the genome given a cmp.h5.
Identifies the coverage contours of the window in order to
identify subintervals where a good consensus can be called.
Creates the desired "no evidence consensus" where there is
inadequate coverage.
"""
winId, winStart, winEnd = refWindow
logging.info("Arrow operating on %s" % reference.windowToString(refWindow))
if options.fancyChunking:
# 1) identify the intervals with adequate coverage for arrow
# consensus; restrict to intervals of length > 10
alnHits = U.readsInWindow(alnFile,
refWindow,
depthLimit=20000,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
starts = np.fromiter((hit.tStart for hit in alnHits), np.int)
ends = np.fromiter((hit.tEnd for hit in alnHits), np.int)
intervals = kSpannedIntervals(refWindow,
arrowConfig.minPoaCoverage,
starts,
ends,
minLength=10)
coverageGaps = holes(refWindow, intervals)
allIntervals = sorted(intervals + coverageGaps)
if len(allIntervals) > 1:
logging.info("Usable coverage in %s: %r" %
(reference.windowToString(refWindow), intervals))
else:
allIntervals = [(winStart, winEnd)]
# 2) pull out the reads we will use for each interval
# 3) call consensusForAlignments on the interval
subConsensi = []
variants = []
for interval in allIntervals:
intStart, intEnd = interval
intRefSeq = referenceContig[intStart:intEnd]
subWin = subWindow(refWindow, interval)
windowRefSeq = referenceContig[intStart:intEnd]
alns = U.readsInWindow(alnFile,
subWin,
depthLimit=depthLimit,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
clippedAlns_ = [aln.clippedTo(*interval) for aln in alns]
clippedAlns = U.filterAlns(subWin, clippedAlns_, arrowConfig)
if len([a for a in clippedAlns if a.spansReferenceRange(*interval)
]) >= arrowConfig.minPoaCoverage:
logging.debug("%s: Reads being used: %s" %
(reference.windowToString(subWin), " ".join(
[str(hit.readName) for hit in alns])))
alnsUsed = [] if options.reportEffectiveCoverage else None
css = U.consensusForAlignments(subWin,
intRefSeq,
clippedAlns,
arrowConfig,
alnsUsed=alnsUsed)
# Tabulate the coverage implied by these alignments, as
# well as the post-filtering ("effective") coverage
siteCoverage = U.coverageInWindow(subWin, alns)
effectiveSiteCoverage = U.coverageInWindow(
subWin, alnsUsed) if options.reportEffectiveCoverage else None
variants_ = U.variantsFromConsensus(subWin,
windowRefSeq,
css.sequence,
css.confidence,
siteCoverage,
effectiveSiteCoverage,
options.aligner,
ai=None)
filteredVars = filterVariants(options.minCoverage,
options.minConfidence, variants_)
# Annotate?
if options.annotateGFF:
annotateVariants(filteredVars, clippedAlns)
variants += filteredVars
# Dump?
maybeDumpEvidence = \
((options.dumpEvidence == "all") or
(options.dumpEvidence == "outliers") or
(options.dumpEvidence == "variants") and (len(variants) > 0))
if maybeDumpEvidence:
refId, refStart, refEnd = subWin
refName = reference.idToName(refId)
windowDirectory = os.path.join(options.evidenceDirectory,
refName,
"%d-%d" % (refStart, refEnd))
ev = ArrowEvidence.fromConsensus(css)
if options.dumpEvidence != "outliers":
ev.save(windowDirectory)
elif (np.max(ev.delta) > 20):
# Mathematically I don't think we should be seeing
# deltas > 6 in magnitude, but let's just restrict
# attention to truly bonkers outliers.
ev.save(windowDirectory)
else:
css = ArrowConsensus.noCallConsensus(
arrowConfig.noEvidenceConsensus, subWin, intRefSeq)
subConsensi.append(css)
# 4) glue the subwindow consensus objects together to form the
# full window consensus
css = join(subConsensi)
# 5) Return
return css, variants
def consensusAndVariantsForWindow(alnFile, refWindow, referenceContig,
depthLimit, arrowConfig):
"""
High-level routine for calling the consensus for a
window of the genome given a cmp.h5.
Identifies the coverage contours of the window in order to
identify subintervals where a good consensus can be called.
Creates the desired "no evidence consensus" where there is
inadequate coverage.
"""
winId, winStart, winEnd = refWindow
logging.info("Arrow operating on %s" %
reference.windowToString(refWindow))
if options.fancyChunking:
# 1) identify the intervals with adequate coverage for arrow
# consensus; restrict to intervals of length > 10
alnHits = U.readsInWindow(alnFile, refWindow,
depthLimit=20000,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
starts = np.fromiter((hit.tStart for hit in alnHits), np.int)
ends = np.fromiter((hit.tEnd for hit in alnHits), np.int)
intervals = kSpannedIntervals(refWindow, arrowConfig.minPoaCoverage,
starts, ends, minLength=10)
coverageGaps = holes(refWindow, intervals)
allIntervals = sorted(intervals + coverageGaps)
if len(allIntervals) > 1:
logging.info("Usable coverage in %s: %r" %
(reference.windowToString(refWindow), intervals))
else:
allIntervals = [ (winStart, winEnd) ]
# 2) pull out the reads we will use for each interval
# 3) call consensusForAlignments on the interval
subConsensi = []
variants = []
for interval in allIntervals:
intStart, intEnd = interval
intRefSeq = referenceContig[intStart:intEnd]
subWin = subWindow(refWindow, interval)
windowRefSeq = referenceContig[intStart:intEnd]
alns = U.readsInWindow(alnFile, subWin,
depthLimit=depthLimit,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
clippedAlns_ = [ aln.clippedTo(*interval) for aln in alns ]
clippedAlns = U.filterAlns(subWin, clippedAlns_, arrowConfig)
if len([ a for a in clippedAlns
if a.spansReferenceRange(*interval) ]) >= arrowConfig.minPoaCoverage:
logging.debug("%s: Reads being used: %s" %
(reference.windowToString(subWin),
" ".join([str(hit.readName) for hit in alns])))
alnsUsed = [] if options.reportEffectiveCoverage else None
css = U.consensusForAlignments(subWin,
intRefSeq,
clippedAlns,
arrowConfig,
alnsUsed=alnsUsed)
# Tabulate the coverage implied by these alignments, as
# well as the post-filtering ("effective") coverage
siteCoverage = U.coverageInWindow(subWin, alns)
effectiveSiteCoverage = U.coverageInWindow(subWin, alnsUsed) if options.reportEffectiveCoverage else None
variants_ = U.variantsFromConsensus(subWin, windowRefSeq,
css.sequence, css.confidence, siteCoverage, effectiveSiteCoverage,
options.aligner,
ai=None)
filteredVars = filterVariants(options.minCoverage,
options.minConfidence,
variants_)
# Annotate?
if options.annotateGFF:
annotateVariants(filteredVars, clippedAlns)
variants += filteredVars
# Dump?
maybeDumpEvidence = \
((options.dumpEvidence == "all") or
(options.dumpEvidence == "outliers") or
(options.dumpEvidence == "variants") and (len(variants) > 0))
if maybeDumpEvidence:
refId, refStart, refEnd = subWin
refName = reference.idToName(refId)
windowDirectory = os.path.join(
options.evidenceDirectory,
refName,
"%d-%d" % (refStart, refEnd))
ev = ArrowEvidence.fromConsensus(css)
if options.dumpEvidence != "outliers":
ev.save(windowDirectory)
elif (np.max(ev.delta) > 20):
# Mathematically I don't think we should be seeing
# deltas > 6 in magnitude, but let's just restrict
# attention to truly bonkers outliers.
ev.save(windowDirectory)
else:
css = ArrowConsensus.noCallConsensus(arrowConfig.noEvidenceConsensus,
subWin, intRefSeq)
subConsensi.append(css)
# 4) glue the subwindow consensus objects together to form the
# full window consensus
css = join(subConsensi)
# 5) Return
return css, variants
def consensusAndVariantsForWindow(alnFile, refWindow, referenceContig,
depthLimit, arrowConfig):
"""
High-level routine for calling the consensus for a
window of the genome given a BAM file.
Identifies the coverage contours of the window in order to
identify subintervals where a good consensus can be called.
Creates the desired "no evidence consensus" where there is
inadequate coverage.
"""
winId, winStart, winEnd = refWindow
logging.info("Arrow operating on %s" %
reference.windowToString(refWindow))
if options.fancyChunking:
# 1) identify the intervals with adequate coverage for arrow
# consensus; restrict to intervals of length > 10
alnHits = U.readsInWindow(alnFile, refWindow,
depthLimit=20000,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
starts = np.fromiter((hit.tStart for hit in alnHits), np.int)
ends = np.fromiter((hit.tEnd for hit in alnHits), np.int)
intervals = kSpannedIntervals(refWindow, arrowConfig.minPoaCoverage,
starts, ends, minLength=10)
coverageGaps = holes(refWindow, intervals)
allIntervals = sorted(intervals + coverageGaps)
if len(allIntervals) > 1:
logging.info("Usable coverage in %s: %r" %
(reference.windowToString(refWindow), intervals))
else:
allIntervals = [ (winStart, winEnd) ]
# 2) pull out the reads we will use for each interval
# 3) call consensusForAlignments on the interval
subConsensi = []
variants = []
for interval in allIntervals:
intStart, intEnd = interval
intRefSeq = referenceContig[intStart:intEnd]
subWin = subWindow(refWindow, interval)
windowRefSeq = referenceContig[intStart:intEnd]
alns = U.readsInWindow(alnFile, subWin,
depthLimit=depthLimit,
minMapQV=arrowConfig.minMapQV,
strategy="long-and-strand-balanced",
stratum=options.readStratum,
barcode=options.barcode)
clippedAlns_ = [ aln.clippedTo(*interval) for aln in alns ]
clippedAlns = U.filterAlns(subWin, clippedAlns_, arrowConfig)
if len([ a for a in clippedAlns
if a.spansReferenceRange(*interval) ]) >= arrowConfig.minPoaCoverage:
logging.debug("%s: Reads being used: %s" %
(reference.windowToString(subWin),
" ".join([str(hit.readName) for hit in alns])))
alnsUsed = [] if options.reportEffectiveCoverage else None
css = U.consensusForAlignments(subWin,
intRefSeq,
clippedAlns,
arrowConfig,
alnsUsed=alnsUsed)
# Tabulate the coverage implied by these alignments, as
# well as the post-filtering ("effective") coverage
siteCoverage = U.coverageInWindow(subWin, alns)
effectiveSiteCoverage = U.coverageInWindow(subWin, alnsUsed) if options.reportEffectiveCoverage else None
variants_, newPureCss = U.variantsFromConsensus(subWin, windowRefSeq, css.sequence, css.confidence,
siteCoverage, effectiveSiteCoverage,
options.aligner, ai=None,
diploid=arrowConfig.polishDiploid)
# Annotate?
if options.annotateGFF:
annotateVariants(variants_, clippedAlns)
variants += variants_
# The nascent consensus sequence might contain ambiguous bases, these
# need to be removed as software in the wild cannot deal with such
# characters and we only use IUPAC for *internal* bookkeeping.
if arrowConfig.polishDiploid:
css.sequence = newPureCss
else:
css = ArrowConsensus.noCallConsensus(arrowConfig.noEvidenceConsensus,
subWin, intRefSeq)
subConsensi.append(css)
# 4) glue the subwindow consensus objects together to form the
# full window consensus
css = join(subConsensi)
# 5) Return
return css, variants