def mismatches(args):
"""
%prog mismatches blastfile
Print out histogram of mismatches of HSPs, usually for evaluating SNP level.
"""
from jcvi.utils.cbook import percentage
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(mismatches.__doc__)
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
blastfile, = args
data = []
matches = 0
b = Blast(blastfile)
for query, bline in b.iter_best_hit():
mm = bline.nmismatch + bline.ngaps
data.append(mm)
nonzeros = [x for x in data if x != 0]
title = "Polymorphic sites: {0}".\
format(percentage(len(nonzeros), len(data)))
stem_leaf_plot(data, 0, 20, 20, title=title)
def count(args):
"""
%prog count cdhit.consensus.fasta
Scan the headers for the consensus clusters and count the number of reads.
"""
from jcvi.formats.fasta import Fasta
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.utils.cbook import SummaryStats
p = OptionParser(count.__doc__)
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
fastafile, = args
f = Fasta(fastafile, lazy=True)
sizes = []
for desc, rec in f.iterdescriptions_ordered():
if desc.startswith("singleton"):
sizes.append(1)
continue
# consensus_for_cluster_0 with 63 sequences
name, w, size, seqs = desc.split()
assert w == "with"
sizes.append(int(size))
s = SummaryStats(sizes)
print >> sys.stderr, s
stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def mismatches(args):
"""
%prog mismatches blastfile
Print out histogram of mismatches of HSPs, usually for evaluating SNP level.
"""
from jcvi.utils.cbook import percentage
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(mismatches.__doc__)
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
blastfile, = args
data = []
b = Blast(blastfile)
for query, bline in b.iter_best_hit():
mm = bline.nmismatch + bline.ngaps
data.append(mm)
nonzeros = [x for x in data if x != 0]
title = "Polymorphic sites: {0}".\
format(percentage(len(nonzeros), len(data)))
stem_leaf_plot(data, 0, 20, 20, title=title)
def count(args):
"""
%prog count cdhit.consensus.fasta
Scan the headers for the consensus clusters and count the number of reads.
"""
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.utils.cbook import SummaryStats
p = OptionParser(count.__doc__)
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
fastafile, = args
f = Fasta(fastafile, lazy=True)
sizes = []
for desc, rec in f.iterdescriptions_ordered():
if desc.startswith("singleton"):
sizes.append(1)
continue
# consensus_for_cluster_0 with 63 sequences
name, w, size, seqs = desc.split()
assert w == "with"
sizes.append(int(size))
s = SummaryStats(sizes)
print >> sys.stderr, s
stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def report(args):
'''
%prog report ksfile
generate a report given a Ks result file (as produced by synonymous_calc.py).
describe the median Ks, Ka values, as well as the distribution in stem-leaf plot
'''
from jcvi.utils.cbook import SummaryStats
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(report.__doc__)
p.add_option("--pdf", default=False, action="store_true",
help="Generate graphic output for the histogram [default: %default]")
p.add_option("--components", default=1, type="int",
help="Number of components to decompose peaks [default: %default]")
add_plot_options(p)
opts, args, iopts = p.set_image_options(args, figsize="5x5")
if len(args) != 1:
sys.exit(not p.print_help())
ks_file, = args
data = read_ks_file(ks_file)
ks_min = opts.vmin
ks_max = opts.vmax
bins = opts.bins
for f in fields.split(",")[1:]:
columndata = [getattr(x, f) for x in data]
ks = ("ks" in f)
if not ks:
continue
columndata = [x for x in columndata if ks_min <= x <= ks_max]
st = SummaryStats(columndata)
title = "{0} ({1}): ".format(descriptions[f], ks_file)
title += "Median:{0:.3f} (1Q:{1:.3f}|3Q:{2:.3f}||".\
format(st.median, st.firstq, st.thirdq)
title += "Mean:{0:.3f}|Std:{1:.3f}||N:{2})".\
format(st.mean, st.sd, st.size)
tbins = (0, ks_max, bins) if ks else (0, .6, 10)
digit = 2 if (ks_max * 1. / bins) < .1 else 1
stem_leaf_plot(columndata, *tbins, digit=digit, title=title)
if not opts.pdf:
return
components = opts.components
data = [x.ng_ks for x in data]
data = [x for x in data if ks_min <= x <= ks_max]
fig = plt.figure(1, (iopts.w, iopts.h))
ax = fig.add_axes([.12, .1, .8, .8])
kp = KsPlot(ax, ks_max, opts.bins, legendp=opts.legendp)
kp.add_data(data, components, fill=opts.fill)
kp.draw(title=opts.title)
def report(args):
'''
%prog report ksfile
generate a report given a Ks result file (as produced by synonymous_calc.py).
describe the median Ks, Ka values, as well as the distribution in stem-leaf plot
'''
from jcvi.utils.cbook import SummaryStats
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(report.__doc__)
p.add_option("--pdf", default=False, action="store_true",
help="Generate graphic output for the histogram [default: %default]")
p.add_option("--components", default=1, type="int",
help="Number of components to decompose peaks [default: %default]")
add_plot_options(p)
opts, args, iopts = p.set_image_options(args, figsize="5x5")
if len(args) != 1:
sys.exit(not p.print_help())
ks_file, = args
data = KsFile(ks_file)
ks_min = opts.vmin
ks_max = opts.vmax
bins = opts.bins
for f in fields.split(",")[1:]:
columndata = [getattr(x, f) for x in data]
ks = ("ks" in f)
if not ks:
continue
columndata = [x for x in columndata if ks_min <= x <= ks_max]
st = SummaryStats(columndata)
title = "{0} ({1}): ".format(descriptions[f], ks_file)
title += "Median:{0:.3f} (1Q:{1:.3f}|3Q:{2:.3f}||".\
format(st.median, st.firstq, st.thirdq)
title += "Mean:{0:.3f}|Std:{1:.3f}||N:{2})".\
format(st.mean, st.sd, st.size)
tbins = (0, ks_max, bins) if ks else (0, .6, 10)
digit = 2 if (ks_max * 1. / bins) < .1 else 1
stem_leaf_plot(columndata, *tbins, digit=digit, title=title)
if not opts.pdf:
return
components = opts.components
data = [x.ng_ks for x in data]
data = [x for x in data if ks_min <= x <= ks_max]
fig = plt.figure(1, (iopts.w, iopts.h))
ax = fig.add_axes([.12, .1, .8, .8])
kp = KsPlot(ax, ks_max, opts.bins, legendp=opts.legendp)
kp.add_data(data, components, fill=opts.fill, fitted=opts.fit)
kp.draw(title=opts.title)
def location(args):
"""
%prog location bedfile fastafile
Given SNP locations, summarize the locations in the sequences. For example,
find out if there are more 3`-SNPs than 5`-SNPs.
"""
from jcvi.formats.bed import BedLine
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(location.__doc__)
p.add_option(
"--dist",
default=100,
type="int",
help="Distance cutoff to call 5` and 3` [default: %default]",
)
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
bedfile, fastafile = args
dist = opts.dist
sizes = Sizes(fastafile).mapping
fp = open(bedfile)
fiveprime = threeprime = total = 0
percentages = []
for row in fp:
b = BedLine(row)
pos = b.start
size = sizes[b.seqid]
if pos < dist:
fiveprime += 1
if size - pos < dist:
threeprime += 1
total += 1
percentages.append(100 * pos / size)
m = "Five prime (within {0}bp of start codon): {1}\n".format(
dist, fiveprime)
m += "Three prime (within {0}bp of stop codon): {1}\n".format(
dist, threeprime)
m += "Total: {0}".format(total)
print(m, file=sys.stderr)
bins = 10
title = "Locations within the gene [0=Five-prime, 100=Three-prime]"
stem_leaf_plot(percentages, 0, 100, bins, title=title)
def count(args):
"""
%prog count cdhit.consensus.fasta
Scan the headers for the consensus clusters and count the number of reads.
"""
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.utils.cbook import SummaryStats
p = OptionParser(count.__doc__)
p.add_option("--csv", help="Write depth per contig to file")
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
(fastafile, ) = args
csv = open(opts.csv, "w") if opts.csv else None
f = Fasta(fastafile, lazy=True)
sizes = []
for desc, rec in f.iterdescriptions_ordered():
if desc.startswith("singleton"):
sizes.append(1)
continue
# consensus_for_cluster_0 with 63 sequences
if "with" in desc:
name, w, size, seqs = desc.split()
if csv:
print("\t".join(str(x) for x in (name, size, len(rec))),
file=csv)
assert w == "with"
sizes.append(int(size))
# MRD85:00603:02472;size=167;
else:
name, size, tail = desc.split(";")
sizes.append(int(size.replace("size=", "")))
if csv:
csv.close()
logging.debug("File written to `%s`.", opts.csv)
s = SummaryStats(sizes)
print(s, file=sys.stderr)
stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def count(args):
"""
%prog count cdhit.consensus.fasta
Scan the headers for the consensus clusters and count the number of reads.
"""
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.utils.cbook import SummaryStats
p = OptionParser(count.__doc__)
p.add_option("--csv", help="Write depth per contig to file")
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
fastafile, = args
csv = open(opts.csv, "w") if opts.csv else None
f = Fasta(fastafile, lazy=True)
sizes = []
for desc, rec in f.iterdescriptions_ordered():
if desc.startswith("singleton"):
sizes.append(1)
continue
# consensus_for_cluster_0 with 63 sequences
if "with" in desc:
name, w, size, seqs = desc.split()
if csv:
print("\t".join(str(x)
for x in (name, size, len(rec))), file=csv)
assert w == "with"
sizes.append(int(size))
# MRD85:00603:02472;size=167;
else:
name, size, tail = desc.split(";")
sizes.append(int(size.replace("size=", "")))
if csv:
csv.close()
logging.debug("File written to `{0}`".format(opts.csv))
s = SummaryStats(sizes)
print(s, file=sys.stderr)
stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def location(args):
"""
%prog location bedfile fastafile
Given SNP locations, summarize the locations in the sequences. For example,
find out if there are more 3`-SNPs than 5`-SNPs.
"""
from jcvi.formats.bed import BedLine
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(location.__doc__)
p.add_option("--dist", default=100, type="int",
help="Distance cutoff to call 5` and 3` [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
bedfile, fastafile = args
dist = opts.dist
sizes = Sizes(fastafile).mapping
fp = open(bedfile)
fiveprime = threeprime = total = 0
percentages = []
for row in fp:
b = BedLine(row)
pos = b.start
size = sizes[b.seqid]
if pos < dist:
fiveprime += 1
if size - pos < dist:
threeprime += 1
total += 1
percentages.append(100 * pos / size)
m = "Five prime (within {0}bp of start codon): {1}\n".format(dist, fiveprime)
m += "Three prime (within {0}bp of stop codon): {1}\n".format(dist, threeprime)
m += "Total: {0}".format(total)
print >> sys.stderr, m
bins = 10
title = "Locations within the gene [0=Five-prime, 100=Three-prime]"
stem_leaf_plot(percentages, 0, 100, bins, title=title)
def htg(args):
"""
%prog htg fastafile template.sbt
Prepare sqnfiles for Genbank HTG submission to update existing records.
`fastafile` contains the records to update, multiple records are allowed
(with each one generating separate sqn file in the sqn/ folder). The record
defline has the accession ID. For example,
>AC148290.3
Internally, this generates two additional files (phasefile and namesfile)
and download records from Genbank. Below is implementation details:
`phasefile` contains, for each accession, phase information. For example:
AC148290.3 3 HTG 2 mth2-45h12
which means this is a Phase-3 BAC. Record with only a single contig will be
labeled as Phase-3 regardless of the info in the `phasefile`. Template file
is the Genbank sbt template. See jcvi.formats.sbt for generation of such
files.
Another problem is that Genbank requires the name of the sequence to stay
the same when updating and will kick back with a table of name conflicts.
For example:
We are unable to process the updates for these entries
for the following reason:
Seqname has changed
Accession Old seq_name New seq_name
--------- ------------ ------------
AC239792 mtg2_29457 AC239792.1
To prepare a submission, this script downloads genbank and asn.1 format,
and generate the phase file and the names file (use formats.agp.phase() and
apps.gbsubmit.asn(), respectively). These get automatically run.
However, use --phases if the genbank files contain outdated information.
For example, the clone name changes or phase upgrades. In this case, run
formats.agp.phase() manually, modify the phasefile and use --phases to override.
"""
from jcvi.formats.fasta import sequin, ids
from jcvi.formats.agp import phase
from jcvi.apps.fetch import entrez
p = OptionParser(htg.__doc__)
p.add_option("--phases", default=None,
help="Use another phasefile to override [default: %default]")
p.add_option("--comment", default="",
help="Comments for this update [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
fastafile, sbtfile = args
pf = fastafile.rsplit(".", 1)[0]
idsfile = pf + ".ids"
phasefile = pf + ".phases"
namesfile = pf + ".names"
ids([fastafile, "--outfile={0}".format(idsfile)])
asndir = "asn.1"
mkdir(asndir)
entrez([idsfile, "--format=asn.1", "--outdir={0}".format(asndir)])
asn(glob("{0}/*".format(asndir)) + \
["--outfile={0}".format(namesfile)])
if opts.phases is None:
gbdir = "gb"
mkdir(gbdir)
entrez([idsfile, "--format=gb", "--outdir={0}".format(gbdir)])
phase(glob("{0}/*".format(gbdir)) + \
["--outfile={0}".format(phasefile)])
else:
phasefile = opts.phases
assert op.exists(namesfile) and op.exists(phasefile)
newphasefile = phasefile + ".new"
newphasefw = open(newphasefile, "w")
comment = opts.comment
fastadir = "fasta"
sqndir = "sqn"
mkdir(fastadir)
mkdir(sqndir)
from jcvi.graphics.histogram import stem_leaf_plot
names = DictFile(namesfile)
assert len(set(names.keys())) == len(set(names.values()))
phases = DictFile(phasefile)
ph = [int(x) for x in phases.values()]
# vmin 1, vmax 4, bins 3
stem_leaf_plot(ph, 1, 4, 3, title="Counts of phases before updates")
logging.debug("Information loaded for {0} records.".format(len(phases)))
assert len(names) == len(phases)
newph = []
cmd = "faSplit byname {0} {1}/".format(fastafile, fastadir)
sh(cmd, outfile="/dev/null", errfile="/dev/null")
acmd = 'tbl2asn -a z -p fasta -r {sqndir}'
acmd += ' -i {splitfile} -t {sbtfile} -C tigr'
acmd += ' -j "{qualifiers}"'
acmd += ' -A {accession_nv} -o {sqndir}/{accession_nv}.sqn -V Vbr'
acmd += ' -y "{comment}" -W T -T T'
qq = "[tech=htgs {phase}] [organism=Medicago truncatula] [strain=A17]"
nupdated = 0
for row in open(phasefile):
atoms = row.rstrip().split("\t")
# see formats.agp.phase() for column contents
accession, phase, clone = atoms[0], atoms[1], atoms[-1]
fafile = op.join(fastadir, accession + ".fa")
accession_nv = accession.split(".", 1)[0]
newid = names[accession_nv]
newidopt = "--newid={0}".format(newid)
cloneopt = "--clone={0}".format(clone)
splitfile, gaps = sequin([fafile, newidopt, cloneopt])
splitfile = op.basename(splitfile)
phase = int(phase)
assert phase in (1, 2, 3)
oldphase = phase
if gaps == 0 and phase != 3:
phase = 3
if gaps != 0 and phase == 3:
phase = 2
print("{0}\t{1}\t{2}".\
format(accession_nv, oldphase, phase), file=newphasefw)
newph.append(phase)
qualifiers = qq.format(phase=phase)
if ";" in clone:
qualifiers += " [keyword=HTGS_POOLED_MULTICLONE]"
cmd = acmd.format(accession=accession, accession_nv=accession_nv,
sqndir=sqndir, sbtfile=sbtfile, splitfile=splitfile,
qualifiers=qualifiers, comment=comment)
sh(cmd)
verify_sqn(sqndir, accession)
nupdated += 1
stem_leaf_plot(newph, 1, 4, 3, title="Counts of phases after updates")
print("A total of {0} records updated.".format(nupdated), file=sys.stderr)
def histogram(args):
"""
%prog histogram [reads.fasta|reads.fastq]
Plot read length distribution for reads. The plot would be similar to the
one generated by SMRT-portal, for example:
http://blog.pacificbiosciences.com/2013/10/data-release-long-read-shotgun.html
Plot has two axes - corresponding to pdf and cdf, respectively. Also adding
number of reads, average/median, N50, and total length.
"""
from jcvi.utils.cbook import human_size, thousands, SUFFIXES
from jcvi.formats.fastq import fasta
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.graphics.base import plt, markup, human_formatter, \
human_base_formatter, savefig, set2, set_ticklabels_helvetica
p = OptionParser(histogram.__doc__)
p.set_histogram(vmax=50000, bins=100, xlabel="Read length",
title="Read length distribution")
p.add_option("--ylabel1", default="Counts",
help="Label of y-axis on the left")
p.add_option("--color", default='0', choices=[str(x) for x in range(8)],
help="Color of bars, which is an index 0-7 in brewer set2")
opts, args, iopts = p.set_image_options(args, figsize="6x6", style="dark")
if len(args) != 1:
sys.exit(not p.print_help())
fastafile, = args
fastafile, qualfile = fasta([fastafile, "--seqtk"])
sizes = Sizes(fastafile)
all_sizes = sorted(sizes.sizes)
xmin, xmax, bins = opts.vmin, opts.vmax, opts.bins
left, height = stem_leaf_plot(all_sizes, xmin, xmax, bins)
plt.figure(1, (iopts.w, iopts.h))
ax1 = plt.gca()
width = (xmax - xmin) * .5 / bins
color = set2[int(opts.color)]
ax1.bar(left, height, width=width, linewidth=0, fc=color, align="center")
ax1.set_xlabel(markup(opts.xlabel))
ax1.set_ylabel(opts.ylabel1)
ax2 = ax1.twinx()
cur_size = 0
total_size, l50, n50 = sizes.summary
cdf = {}
hsize = human_size(total_size)
tag = hsize[-2:]
unit = 1000 ** SUFFIXES[1000].index(tag)
for x in all_sizes:
if x not in cdf:
cdf[x] = (total_size - cur_size) * 1. / unit
cur_size += x
x, y = zip(*sorted(cdf.items()))
ax2.plot(x, y, '-', color="darkslategray")
ylabel2 = "{0} above read length".format(tag)
ax2.set_ylabel(ylabel2)
for ax in (ax1, ax2):
set_ticklabels_helvetica(ax)
ax.set_xlim((xmin - width / 2, xmax + width / 2))
tc = "gray"
axt = ax1.transAxes
xx, yy = .95, .95
ma = "Total bases: {0}".format(hsize)
mb = "Total reads: {0}".format(thousands(len(sizes)))
mc = "Average read length: {0}bp".format(thousands(np.mean(all_sizes)))
md = "Median read length: {0}bp".format(thousands(np.median(all_sizes)))
me = "N50 read length: {0}bp".format(thousands(l50))
for t in (ma, mb, mc, md, me):
print >> sys.stderr, t
ax1.text(xx, yy, t, color=tc, transform=axt, ha="right")
yy -= .05
ax1.set_title(markup(opts.title))
# Seaborn removes ticks for all styles except 'ticks'. Now add them back:
ax1.tick_params(axis="x", direction="out", length=3,
left=False, right=False, top=False, bottom=True)
ax1.xaxis.set_major_formatter(human_base_formatter)
ax1.yaxis.set_major_formatter(human_formatter)
figname = sizes.filename + ".pdf"
savefig(figname)
def histogram(args):
"""
%prog histogram [reads.fasta|reads.fastq]
Plot read length distribution for reads. The plot would be similar to the
one generated by SMRT-portal, for example:
http://blog.pacificbiosciences.com/2013/10/data-release-long-read-shotgun.html
Plot has two axes - corresponding to pdf and cdf, respectively. Also adding
number of reads, average/median, N50, and total length.
"""
from jcvi.utils.cbook import human_size, thousands, SUFFIXES
from jcvi.formats.fastq import fasta
from jcvi.graphics.histogram import stem_leaf_plot
from jcvi.graphics.base import (
plt,
markup,
human_formatter,
human_base_formatter,
savefig,
set2,
set_ticklabels_helvetica,
)
p = OptionParser(histogram.__doc__)
p.set_histogram(vmax=50000,
bins=100,
xlabel="Read length",
title="Read length distribution")
p.add_option("--ylabel1",
default="Counts",
help="Label of y-axis on the left")
p.add_option(
"--color",
default="0",
choices=[str(x) for x in range(8)],
help="Color of bars, which is an index 0-7 in brewer set2",
)
opts, args, iopts = p.set_image_options(args, figsize="6x6", style="dark")
if len(args) != 1:
sys.exit(not p.print_help())
(fastafile, ) = args
fastafile, qualfile = fasta([fastafile, "--seqtk"])
sizes = Sizes(fastafile)
all_sizes = sorted(sizes.sizes)
xmin, xmax, bins = opts.vmin, opts.vmax, opts.bins
left, height = stem_leaf_plot(all_sizes, xmin, xmax, bins)
plt.figure(1, (iopts.w, iopts.h))
ax1 = plt.gca()
width = (xmax - xmin) * 0.5 / bins
color = set2[int(opts.color)]
ax1.bar(left, height, width=width, linewidth=0, fc=color, align="center")
ax1.set_xlabel(markup(opts.xlabel))
ax1.set_ylabel(opts.ylabel1)
ax2 = ax1.twinx()
cur_size = 0
total_size, l50, n50 = sizes.summary
cdf = {}
hsize = human_size(total_size)
tag = hsize[-2:]
unit = 1000**SUFFIXES[1000].index(tag)
for x in all_sizes:
if x not in cdf:
cdf[x] = (total_size - cur_size) * 1.0 / unit
cur_size += x
x, y = zip(*sorted(cdf.items()))
ax2.plot(x, y, "-", color="darkslategray")
ylabel2 = "{0} above read length".format(tag)
ax2.set_ylabel(ylabel2)
for ax in (ax1, ax2):
set_ticklabels_helvetica(ax)
ax.set_xlim((xmin - width / 2, xmax + width / 2))
tc = "gray"
axt = ax1.transAxes
xx, yy = 0.95, 0.95
ma = "Total bases: {0}".format(hsize)
mb = "Total reads: {0}".format(thousands(len(sizes)))
mc = "Average read length: {0}bp".format(thousands(np.mean(all_sizes)))
md = "Median read length: {0}bp".format(thousands(np.median(all_sizes)))
me = "N50 read length: {0}bp".format(thousands(l50))
for t in (ma, mb, mc, md, me):
print(t, file=sys.stderr)
ax1.text(xx, yy, t, color=tc, transform=axt, ha="right")
yy -= 0.05
ax1.set_title(markup(opts.title))
# Seaborn removes ticks for all styles except 'ticks'. Now add them back:
ax1.tick_params(
axis="x",
direction="out",
length=3,
left=False,
right=False,
top=False,
bottom=True,
)
ax1.xaxis.set_major_formatter(human_base_formatter)
ax1.yaxis.set_major_formatter(human_formatter)
figname = sizes.filename + ".pdf"
savefig(figname)
def report(args):
'''
%prog report ksfile
generate a report given a Ks result file (as produced by synonymous_calc.py).
describe the median Ks, Ka values, as well as the distribution in stem-leaf plot
'''
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(report.__doc__)
p.add_option("--vmax", default=2., type="float",
help="Maximum value, inclusive [default: %default]")
p.add_option("--bins", default=20, type="int",
help="Number of bins to plot in the histogram [default: %default]")
p.add_option("--pdf", default=False, action="store_true",
help="Generate graphic output for the histogram [default: %default]")
p.add_option("--components", default=1, type="int",
help="Number of components to decompose peaks [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
ks_file, = args
header, data = read_ks_file(ks_file)
ks_max = opts.vmax
for f in fields.split()[1:]:
columndata = [getattr(x, f) for x in data]
title = "{0}: {1:.2f}".format(descriptions[f], np.median(columndata))
title += " ({0:.2f} +/- {1:.2f})".\
format(np.mean(columndata), np.std(columndata))
ks = ("ks" in f)
if not ks:
continue
bins = (0, ks_max, opts.bins) if ks else (0, .6, 10)
digit = 1 if ks else 2
stem_leaf_plot(columndata, *bins, digit=digit, title=title)
if not opts.pdf:
return
from jcvi.graphics.base import mpl, _, tex_formatter, tex_1digit_formatter
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
fig = mpl.figure.Figure(figsize=(5, 5))
canvas = FigureCanvas(fig)
ax = fig.add_axes([.12, .1, .8, .8])
components = opts.components
data = [x.ng_ks for x in data]
interval = ks_max / opts.bins
line, line_mixture = plot_ks_dist(ax, data, interval, components, ks_max, color='r')
leg = ax.legend((line, line_mixture), ("Ks", "Ks (fitted)"),
shadow=True, fancybox=True, prop={"size": 10})
leg.get_frame().set_alpha(.5)
ax.set_xlim((0, ks_max))
ax.set_title(_('Ks distribution'), fontweight="bold")
ax.set_xlabel(_('Synonymous substitutions per site (Ks)'))
ax.set_ylabel(_('Percentage of gene pairs'))
ax.xaxis.set_major_formatter(tex_1digit_formatter)
ax.yaxis.set_major_formatter(tex_formatter)
image_name = "Ks_plot.pdf"
canvas.print_figure(image_name, dpi=300)
logging.debug("Print image to `{0}`.".format(image_name))
def htg(args):
"""
%prog htg fastafile template.sbt
Prepare sqnfiles for Genbank HTG submission to update existing records.
`fastafile` contains the records to update, multiple records are allowed
(with each one generating separate sqn file in the sqn/ folder). The record
defline has the accession ID. For example,
>AC148290.3
Internally, this generates two additional files (phasefile and namesfile)
and download records from Genbank. Below is implementation details:
`phasefile` contains, for each accession, phase information. For example:
AC148290.3 3 HTG 2 mth2-45h12
which means this is a Phase-3 BAC. Record with only a single contig will be
labeled as Phase-3 regardless of the info in the `phasefile`. Template file
is the Genbank sbt template. See jcvi.formats.sbt for generation of such
files.
Another problem is that Genbank requires the name of the sequence to stay
the same when updating and will kick back with a table of name conflicts.
For example:
We are unable to process the updates for these entries
for the following reason:
Seqname has changed
Accession Old seq_name New seq_name
--------- ------------ ------------
AC239792 mtg2_29457 AC239792.1
To prepare a submission, this script downloads genbank and asn.1 format,
and generate the phase file and the names file (use formats.agp.phase() and
apps.gbsubmit.asn(), respectively). These get automatically run.
However, use --phases if the genbank files contain outdated information.
For example, the clone name changes or phase upgrades. In this case, run
formats.agp.phase() manually, modify the phasefile and use --phases to override.
"""
from jcvi.formats.fasta import sequin, ids
from jcvi.formats.agp import phase
from jcvi.apps.fetch import entrez
p = OptionParser(htg.__doc__)
p.add_option(
"--phases",
default=None,
help="Use another phasefile to override",
)
p.add_option("--comment", default="", help="Comments for this update")
opts, args = p.parse_args(args)
if len(args) != 2:
sys.exit(not p.print_help())
fastafile, sbtfile = args
pf = fastafile.rsplit(".", 1)[0]
idsfile = pf + ".ids"
phasefile = pf + ".phases"
namesfile = pf + ".names"
ids([fastafile, "--outfile={0}".format(idsfile)])
asndir = "asn.1"
mkdir(asndir)
entrez([idsfile, "--format=asn.1", "--outdir={0}".format(asndir)])
asn(glob("{0}/*".format(asndir)) + ["--outfile={0}".format(namesfile)])
if opts.phases is None:
gbdir = "gb"
mkdir(gbdir)
entrez([idsfile, "--format=gb", "--outdir={0}".format(gbdir)])
phase(
glob("{0}/*".format(gbdir)) + ["--outfile={0}".format(phasefile)])
else:
phasefile = opts.phases
assert op.exists(namesfile) and op.exists(phasefile)
newphasefile = phasefile + ".new"
newphasefw = open(newphasefile, "w")
comment = opts.comment
fastadir = "fasta"
sqndir = "sqn"
mkdir(fastadir)
mkdir(sqndir)
from jcvi.graphics.histogram import stem_leaf_plot
names = DictFile(namesfile)
assert len(set(names.keys())) == len(set(names.values()))
phases = DictFile(phasefile)
ph = [int(x) for x in phases.values()]
# vmin 1, vmax 4, bins 3
stem_leaf_plot(ph, 1, 4, 3, title="Counts of phases before updates")
logging.debug("Information loaded for {0} records.".format(len(phases)))
assert len(names) == len(phases)
newph = []
cmd = "faSplit byname {0} {1}/".format(fastafile, fastadir)
sh(cmd, outfile="/dev/null", errfile="/dev/null")
acmd = "tbl2asn -a z -p fasta -r {sqndir}"
acmd += " -i {splitfile} -t {sbtfile} -C tigr"
acmd += ' -j "{qualifiers}"'
acmd += " -A {accession_nv} -o {sqndir}/{accession_nv}.sqn -V Vbr"
acmd += ' -y "{comment}" -W T -T T'
qq = "[tech=htgs {phase}] [organism=Medicago truncatula] [strain=A17]"
nupdated = 0
for row in open(phasefile):
atoms = row.rstrip().split("\t")
# see formats.agp.phase() for column contents
accession, phase, clone = atoms[0], atoms[1], atoms[-1]
fafile = op.join(fastadir, accession + ".fa")
accession_nv = accession.split(".", 1)[0]
newid = names[accession_nv]
newidopt = "--newid={0}".format(newid)
cloneopt = "--clone={0}".format(clone)
splitfile, gaps = sequin([fafile, newidopt, cloneopt])
splitfile = op.basename(splitfile)
phase = int(phase)
assert phase in (1, 2, 3)
oldphase = phase
if gaps == 0 and phase != 3:
phase = 3
if gaps != 0 and phase == 3:
phase = 2
print("{0}\t{1}\t{2}".format(accession_nv, oldphase, phase),
file=newphasefw)
newph.append(phase)
qualifiers = qq.format(phase=phase)
if ";" in clone:
qualifiers += " [keyword=HTGS_POOLED_MULTICLONE]"
cmd = acmd.format(
accession=accession,
accession_nv=accession_nv,
sqndir=sqndir,
sbtfile=sbtfile,
splitfile=splitfile,
qualifiers=qualifiers,
comment=comment,
)
sh(cmd)
verify_sqn(sqndir, accession)
nupdated += 1
stem_leaf_plot(newph, 1, 4, 3, title="Counts of phases after updates")
print("A total of {0} records updated.".format(nupdated), file=sys.stderr)
def report(args):
'''
%prog report ksfile
generate a report given a Ks result file (as produced by synonymous_calc.py).
describe the median Ks, Ka values, as well as the distribution in stem-leaf plot
'''
from jcvi.graphics.histogram import stem_leaf_plot
p = OptionParser(report.__doc__)
p.add_option("--vmax",
default=2.,
type="float",
help="Maximum value, inclusive [default: %default]")
p.add_option(
"--bins",
default=20,
type="int",
help="Number of bins to plot in the histogram [default: %default]")
p.add_option(
"--pdf",
default=False,
action="store_true",
help="Generate graphic output for the histogram [default: %default]")
p.add_option(
"--components",
default=1,
type="int",
help="Number of components to decompose peaks [default: %default]")
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
ks_file, = args
header, data = read_ks_file(ks_file)
ks_max = opts.vmax
for f in fields.split()[1:]:
columndata = [getattr(x, f) for x in data]
title = "{0}: {1:.2f}".format(descriptions[f], np.median(columndata))
title += " ({0:.2f} +/- {1:.2f})".\
format(np.mean(columndata), np.std(columndata))
ks = ("ks" in f)
if not ks:
continue
bins = (0, ks_max, opts.bins) if ks else (0, .6, 10)
digit = 1 if ks else 2
stem_leaf_plot(columndata, *bins, digit=digit, title=title)
if not opts.pdf:
return
from jcvi.graphics.base import mpl, _, tex_formatter, tex_1digit_formatter
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
fig = mpl.figure.Figure(figsize=(5, 5))
canvas = FigureCanvas(fig)
ax = fig.add_axes([.12, .1, .8, .8])
components = opts.components
data = [x.ng_ks for x in data]
interval = ks_max / opts.bins
line, line_mixture = plot_ks_dist(ax,
data,
interval,
components,
ks_max,
color='r')
leg = ax.legend((line, line_mixture), ("Ks", "Ks (fitted)"),
shadow=True,
fancybox=True,
prop={"size": 10})
leg.get_frame().set_alpha(.5)
ax.set_xlim((0, ks_max))
ax.set_title(_('Ks distribution'), fontweight="bold")
ax.set_xlabel(_('Synonymous substitutions per site (Ks)'))
ax.set_ylabel(_('Percentage of gene pairs'))
ax.xaxis.set_major_formatter(tex_1digit_formatter)
ax.yaxis.set_major_formatter(tex_formatter)
image_name = "Ks_plot.pdf"
canvas.print_figure(image_name, dpi=300)
logging.debug("Print image to `{0}`.".format(image_name))
