def blast_to_twobeds(blastfile, rclip=1):
key1 = lambda x: x.query
key2 = lambda x: x.query[:-rclip] if rclip else key1
data = BlastSlow(blastfile)
OK = "OK"
fw = open("after.bed", "w")
fwlabels = open("after.labels", "w")
for pe, lines in groupby(data, key=key2):
label = OK
lines = list(lines)
assert len(lines) in (1, 2)
if len(lines) != 2:
label = "Singleton"
else:
a, b = lines
aquery, bquery = a.query, b.query
asubject, bsubject = a.subject, b.subject
if asubject != bsubject:
label = "Different chr {0}|{1}".format(asubject, bsubject)
else:
astrand, bstrand = a.orientation, b.orientation
assert aquery[-1] == "L" and bquery[-1] == "R", str(
(aquery, bquery))
if astrand == "+" and bstrand == "+":
sstart, sstop = a.sstop + 1, b.sstart - 1
elif astrand == "-" and bstrand == "-":
sstart, sstop = b.sstop + 1, a.sstart - 1
else:
label = "Strand {0}|{1}".format(astrand, bstrand)
if label == OK:
strand = "+"
label = sstop - sstart + 1
if sstart > sstop:
sstart, sstop = sstop, sstart
strand = "-"
label = -(sstop - sstart + 1)
print(
"\t".join(
str(x) for x in (asubject, sstart - 1, sstop, pe, strand)),
file=fw,
)
print("\t".join(str(x) for x in (pe, label)), file=fwlabels)
fw.close()
fwlabels.close()
return fwlabels.name
def blast(self, blastfile=None, outfile=None):
"""
convert anchor file to 12 col blast file
"""
from jcvi.formats.blast import BlastSlow, BlastLineByConversion
if not outfile:
outfile = self.filename + ".blast"
if blastfile is not None:
blasts = BlastSlow(blastfile).to_dict()
else:
blasts = None
fw = must_open(outfile, "w", checkexists=True)
nlines = 0
for a, b, id in self.iter_pairs():
if (a, b) in blasts:
bline = blasts[(a, b)]
elif (b, a) in blasts:
bline = blasts[(b, a)]
else:
line = "\t".join((a, b))
bline = BlastLineByConversion(line, mode="110000000000")
print >> fw, bline
nlines += 1
fw.close()
logging.debug("A total of {0} BLAST lines written to `{1}`."\
.format(nlines, outfile))
return outfile
def fromblast(args):
"""
%prog fromblast blastfile subject.fasta
Generate path from BLAST file. If multiple subjects map to the same query,
an edge is constructed between them (with the link provided by the query).
The BLAST file MUST be filtered, chained, supermapped.
"""
from jcvi.formats.blast import sort
from jcvi.utils.range import range_distance
p = OptionParser(fromblast.__doc__)
p.add_option(
"--clique",
default=False,
action="store_true",
help="Populate clique instead of linear path",
)
p.add_option(
"--maxdist",
default=100000,
type="int",
help="Create edge within certain distance",
)
p.set_verbose(help="Print verbose reports to stdout")
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
blastfile, subjectfasta = args
clique = opts.clique
maxdist = opts.maxdist
sort([blastfile, "--query"])
blast = BlastSlow(blastfile, sorted=True)
g = BiGraph()
for query, blines in groupby(blast, key=lambda x: x.query):
blines = list(blines)
iterator = combinations(blines, 2) if clique else pairwise(blines)
for a, b in iterator:
asub, bsub = a.subject, b.subject
if asub == bsub:
continue
arange = (a.query, a.qstart, a.qstop, "+")
brange = (b.query, b.qstart, b.qstop, "+")
dist, oo = range_distance(arange, brange, distmode="ee")
if dist > maxdist:
continue
atag = ">" if a.orientation == "+" else "<"
btag = ">" if b.orientation == "+" else "<"
g.add_edge(asub, bsub, atag, btag)
graph_to_agp(g, blastfile, subjectfasta, verbose=opts.verbose)
def blast_to_twobeds(
blastfile, order, log=False, rclip=1, maxsize=300000, flipbeds=False
):
abed, bbed = "before.bed", "after.bed"
beforebed, afterbed = abed, bbed
if flipbeds:
beforebed, afterbed = afterbed, beforebed
fwa = open(beforebed, "w")
fwb = open(afterbed, "w")
if log:
logfile = "problems.log"
log = open(logfile, "w")
key1 = lambda x: x.query
key2 = lambda x: x.query[:-rclip] if rclip else key1
data = BlastSlow(blastfile)
OK = "OK"
seen = set()
for pe, lines in groupby(data, key=key2):
label = OK
lines = list(lines)
if len(lines) != 2:
label = "Singleton"
else:
a, b = lines
aquery, bquery = a.query, b.query
asubject, bsubject = a.subject, b.subject
if asubject != bsubject:
label = "Different chr {0}|{1}".format(asubject, bsubject)
else:
astrand, bstrand = a.orientation, b.orientation
assert aquery[-1] == "L" and bquery[-1] == "R", str((aquery, bquery))
ai, ax = order[aquery]
bi, bx = order[bquery]
qstart, qstop = ax.start + a.qstart - 1, bx.start + b.qstop - 1
if astrand == "+" and bstrand == "+":
sstart, sstop = a.sstart, b.sstop
elif astrand == "-" and bstrand == "-":
sstart, sstop = b.sstart, a.sstop
else:
label = "Strand {0}|{1}".format(astrand, bstrand)
if sstart > sstop:
label = "Start beyond stop"
if sstop > sstart + maxsize:
label = "Stop beyond start plus {0}".format(maxsize)
aquery = lines[0].query
bac_name = aquery[:-1]
seen.add(bac_name)
name = bac_name + "LR"
if label != OK:
if log:
print("\t".join((name, label)), file=log)
continue
print(
"\t".join(str(x) for x in (ax.seqid, qstart - 1, qstop, name, 1000, "+")),
file=fwa,
)
print(
"\t".join(
str(x) for x in (asubject, sstart - 1, sstop, name, 1000, astrand)
),
file=fwb,
)
# Missing
if log:
label = "Missing"
for k in order.keys():
k = k[:-1]
if k not in seen:
seen.add(k)
k += "LR"
print("\t".join((k, label)), file=log)
log.close()
fwa.close()
fwb.close()
return abed, bbed
def anneal(args):
"""
%prog anneal agpfile contigs.fasta
Merge adjacent overlapping contigs and make new AGP file.
By default it will also anneal lines like these together (unless --nozipshreds):
scaffold4 1 1608 1 W ca-bacs.5638.frag11.22000-23608 1 1608 -
scaffold4 1609 1771 2 N 163 scaffold yes paired-ends
scaffold4 1772 3771 3 W ca-bacs.5638.frag10.20000-22000 1 2000 -
These are most likely shreds, which we look for based on names.
"""
p = OptionParser(anneal.__doc__)
p.set_align(pctid=GoodPct, hitlen=GoodOverlap)
p.add_option("--hang",
default=GoodOverhang,
type="int",
help="Maximum overhang length [default: %default]")
p.set_outdir(outdir="outdir")
p.set_cpus()
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
agpfile, contigs = args
outdir = opts.outdir
if not op.exists(outdir):
mkdir(outdir)
cmd = "faSplit byname {0} {1}/".format(contigs, outdir)
sh(cmd)
cutoff = Cutoff(opts.pctid, opts.hitlen, opts.hang)
logging.debug(str(cutoff))
agp = AGP(agpfile)
blastfile = agpfile.replace(".agp", ".blast")
if not op.exists(blastfile):
populate_blastfile(blastfile, agp, outdir, opts)
assert op.exists(blastfile)
logging.debug("File `{0}` found. Start loading.".format(blastfile))
blast = BlastSlow(blastfile).to_dict()
annealedagp = "annealed.agp"
annealedfasta = "annealed.fasta"
newagp = deepcopy(agp)
clrstore = {}
for a, b, qreverse in agp.iter_paired_components():
aid = a.component_id
bid = b.component_id
pair = (aid, bid)
if pair in blast:
bl = blast[pair]
else:
oopts = get_overlap_opts(aid, bid, qreverse, outdir, opts)
o = overlap(oopts)
if not o:
continue
bl = o.blastline
o = Overlap(bl,
a.component_span,
b.component_span,
cutoff,
qreverse=qreverse)
if aid not in clrstore:
clrstore[aid] = CLR.from_agpline(a)
if bid not in clrstore:
clrstore[bid] = CLR.from_agpline(b)
aclr, bclr = clrstore[aid], clrstore[bid]
o.print_graphic()
if o.anneal(aclr, bclr):
newagp.delete_between(aid, bid, verbose=True)
if o.otype == 2: # b ~ a
o = o.swapped
o.print_graphic()
if o.anneal(bclr, aclr):
newagp.switch_between(bid, aid, verbose=True)
newagp.delete_between(bid, aid, verbose=True)
logging.debug("A total of {0} components with modified CLR.".\
format(len(clrstore)))
for cid, c in clrstore.items():
if c.is_valid:
continue
print >> sys.stderr, "Remove {0}".format(c)
newagp.convert_to_gap(cid, verbose=True)
# Update all ranges that has modified clr
for a in newagp:
if a.is_gap:
continue
aid = a.component_id
if aid in clrstore:
c = clrstore[aid]
a.component_beg = c.start
a.component_end = c.end
newagp.print_to_file(annealedagp)
tidyagp = tidy([annealedagp, contigs])
build([tidyagp, contigs, annealedfasta])
return annealedfasta
def fromblast(args):
"""
%prog fromblast blastfile subject.fasta
Generate path from BLAST file. If multiple subjects map to the same query,
an edge is constructed between them (with the link provided by the query).
The BLAST file MUST be filtered, chained, supermapped.
"""
from jcvi.formats.blast import sort
from jcvi.utils.range import range_distance
p = OptionParser(fromblast.__doc__)
p.add_option(
"--clique",
default=False,
action="store_true",
help="Populate clique instead of linear path [default: %default]")
p.add_option(
"--maxdist",
default=100000,
type="int",
help="Create edge within certain distance [default: %default]")
p.add_option("--verbose",
default=False,
action="store_true",
help="Print verbose reports to stdout [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
blastfile, subjectfasta = args
clique = opts.clique
maxdist = opts.maxdist
sort([blastfile, "--query"])
blast = BlastSlow(blastfile, sorted=True)
g = BiGraph()
for query, blines in groupby(blast, key=lambda x: x.query):
blines = list(blines)
iterator = combinations(blines, 2) if clique else pairwise(blines)
for a, b in iterator:
asub, bsub = a.subject, b.subject
if asub == bsub:
continue
arange = (a.query, a.qstart, a.qstop, "+")
brange = (b.query, b.qstart, b.qstop, "+")
dist, oo = range_distance(arange, brange, distmode="ee")
if dist > maxdist:
continue
atag = ">" if a.orientation == "+" else "<"
btag = ">" if b.orientation == "+" else "<"
g.add_edge(BiEdge(asub, bsub, atag, btag))
g.write("graph.txt")
#g.draw("graph.pdf")
logging.debug(str(g))
paths = []
for path in g.iter_paths():
m, oo = g.path(path)
if len(oo) == 1: # Singleton path
continue
paths.append(oo)
if opts.verbose:
print m
print oo
npaths = len(paths)
ntigs = sum(len(x) for x in paths)
logging.debug("Graph decomposed to {0} paths with {1} components.".\
format(npaths, ntigs))
agpfile = blastfile + ".agp"
sizes = Sizes(subjectfasta)
fwagp = open(agpfile, "w")
scaffolded = set()
for i, oo in enumerate(paths):
ctgorder = [(str(ctg), ("+" if strand else "-")) \
for ctg, strand in oo]
scaffolded |= set(ctg for ctg, strand in ctgorder)
object = "pmol_{0:04d}".format(i)
order_to_agp(object, ctgorder, sizes.mapping, fwagp)
# Get the singletons as well
nsingletons = 0
for ctg, size in sizes.iter_sizes():
if ctg in scaffolded:
continue
ctgorder = [(ctg, "+")]
object = ctg
order_to_agp(object, ctgorder, sizes.mapping, fwagp)
nsingletons += 1
logging.debug("Written {0} unscaffolded singletons.".format(nsingletons))
fwagp.close()
logging.debug("AGP file written to `{0}`.".format(agpfile))
def install(args):
"""
%prog install patchers.bed patchers.fasta backbone.fasta alt.fasta
Install patches into backbone, using sequences from alternative assembly.
The patches sequences are generated via jcvi.assembly.patch.fill().
The output is a bedfile that can be converted to AGP using
jcvi.formats.agp.frombed().
"""
from jcvi.apps.base import blast
from jcvi.formats.blast import BlastSlow
from jcvi.formats.fasta import SeqIO
from jcvi.utils.iter import roundrobin
p = OptionParser(install.__doc__)
p.add_option(
"--rclip",
default=1,
type="int",
help="Pair ID is derived from rstrip N chars [default: %default]")
p.add_option(
"--maxsize",
default=1000000,
type="int",
help="Maximum size of patchers to be replaced [default: %default]")
p.add_option("--prefix",
help="Prefix of the new object [default: %default]")
p.add_option(
"--strict",
default=False,
action="store_true",
help="Only update if replacement has no gaps [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 4:
sys.exit(not p.print_help())
pbed, pfasta, bbfasta, altfasta = args
Max = opts.maxsize # Max DNA size to replace gap
rclip = opts.rclip
prefix = opts.prefix
blastfile = blast([altfasta, pfasta, "--wordsize=100", "--pctid=99"])
order = Bed(pbed).order
beforebed, afterbed = "before.bed", "after.bed"
fwa = open(beforebed, "w")
fwb = open(afterbed, "w")
key1 = lambda x: x.query
key2 = lambda x: x.query[:-rclip] if rclip else key1
data = BlastSlow(blastfile)
for pe, lines in groupby(data, key=key2):
lines = list(lines)
if len(lines) != 2:
continue
a, b = lines
aquery, bquery = a.query, b.query
asubject, bsubject = a.subject, b.subject
if asubject != bsubject:
continue
astrand, bstrand = a.orientation, b.orientation
assert aquery[-1] == 'L' and bquery[-1] == 'R', str((aquery, bquery))
ai, ax = order[aquery]
bi, bx = order[bquery]
qstart, qstop = ax.start + a.qstart - 1, bx.start + b.qstop - 1
if astrand == '+' and bstrand == '+':
sstart, sstop = a.sstart, b.sstop
elif astrand == '-' and bstrand == '-':
sstart, sstop = b.sstart, a.sstop
else:
continue
if sstart > sstop:
continue
if sstop > sstart + Max:
continue
name = aquery[:-1] + "LR"
print >> fwa, "\t".join(str(x) for x in \
(ax.seqid, qstart - 1, qstop, name, 1000, "+"))
print >> fwb, "\t".join(str(x) for x in \
(asubject, sstart - 1, sstop, name, 1000, astrand))
fwa.close()
fwb.close()
beforefasta = fastaFromBed(beforebed, bbfasta, name=True, stranded=True)
afterfasta = fastaFromBed(afterbed, altfasta, name=True, stranded=True)
# Exclude the replacements that contain more Ns than before
ah = SeqIO.parse(beforefasta, "fasta")
bh = SeqIO.parse(afterfasta, "fasta")
count_Ns = lambda x: x.seq.count('n') + x.seq.count('N')
exclude = set()
for arec, brec in zip(ah, bh):
an = count_Ns(arec)
bn = count_Ns(brec)
if opts.strict:
if bn == 0:
continue
elif bn < an:
continue
id = arec.id
exclude.add(id)
logging.debug("Ignore {0} updates because of decreasing quality."\
.format(len(exclude)))
abed = Bed(beforebed, sorted=False)
bbed = Bed(afterbed, sorted=False)
abed = [x for x in abed if x.accn not in exclude]
bbed = [x for x in bbed if x.accn not in exclude]
abedfile = "before.filtered.bed"
bbedfile = "after.filtered.bed"
afbed = Bed()
afbed.extend(abed)
bfbed = Bed()
bfbed.extend(bbed)
afbed.print_to_file(abedfile)
bfbed.print_to_file(bbedfile)
# Shuffle the two bedfiles together
sz = Sizes(bbfasta)
sizes = sz.mapping
shuffled = "shuffled.bed"
border = bfbed.order
all = []
afbed.sort(key=afbed.nullkey)
totalids = len(sizes)
import math
pad = int(math.log10(totalids)) + 1
cj = 0
seen = set()
accn = lambda x: "{0}{1:0{2}d}".format(prefix, x, pad)
for seqid, aa in afbed.sub_beds():
cj += 1
abeds, bbeds, beds = [], [], []
size = sizes[seqid]
ranges = [(x.seqid, x.start, x.end) for x in aa]
cranges = range_interleave(ranges, sizes={seqid: size})
for seqid, start, end in cranges:
bedline = "\t".join(str(x) for x in (seqid, start - 1, end))
abeds.append(BedLine(bedline))
for a in aa:
gapid = a.accn
bi, b = border[gapid]
bbeds.append(b)
a = abeds[0] if abeds else []
assert abs(len(abeds) - len(bbeds)) <= 1
if (not a) or a.start > 1:
abeds, bbeds = bbeds, abeds
beds = list(roundrobin(abeds, bbeds))
if prefix:
for b in beds:
b.accn = accn(cj)
all.extend(beds)
seen.add(seqid)
# Singletons
for seqid, size in sz.iter_sizes():
if seqid in seen:
continue
bedline = "\t".join(str(x) for x in (seqid, 0, size, accn(cj)))
b = BedLine(bedline)
cj += 1
if prefix:
b.accn = accn(cj)
all.append(b)
shuffledbed = Bed()
shuffledbed.extend(all)
shuffledbed.print_to_file(shuffled)