def frombed(args): """ %prog frombed bedfile contigfasta readfasta Convert read placement to contig format. This is useful before running BAMBUS. """ from jcvi.formats.fasta import Fasta, SeqIO from jcvi.formats.bed import Bed from jcvi.utils.cbook import fill p = OptionParser(frombed.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) bedfile, contigfasta, readfasta = args prefix = bedfile.rsplit(".", 1)[0] contigfile = prefix + ".contig" idsfile = prefix + ".ids" contigfasta = Fasta(contigfasta) readfasta = Fasta(readfasta) bed = Bed(bedfile) checksum = "00000000 checksum." fw_ids = open(idsfile, "w") fw = open(contigfile, "w") for ctg, reads in bed.sub_beds(): ctgseq = contigfasta[ctg] ctgline = "##{0} {1} {2} bases, {3}".format(\ ctg, len(reads), len(ctgseq), checksum) print >> fw_ids, ctg print >> fw, ctgline print >> fw, fill(ctgseq.seq) for b in reads: read = b.accn strand = b.strand readseq = readfasta[read] rc = " [RC]" if strand == "-" else "" readlen = len(readseq) rstart, rend = 1, readlen if strand == "-": rstart, rend = rend, rstart readrange = "{{{0} {1}}}".format(rstart, rend) conrange = "<{0} {1}>".format(b.start, b.end) readline = "#{0}(0){1} {2} bases, {3} {4} {5}".format(\ read, rc, readlen, checksum, readrange, conrange) print >> fw, readline print >> fw, fill(readseq.seq) logging.debug("Mapped contigs written to `{0}`.".format(contigfile)) logging.debug("Contig IDs written to `{0}`.".format(idsfile))
def prepare(args): """ %prog prepare pairsfile cdsfile [pepfile] -o paired.cds.fasta Pick sequences from cdsfile to form pairs, ready to be calculated. The pairsfile can be generated from formats.blast.cscore(). The first two columns contain the pair. """ from jcvi.formats.fasta import Fasta p = OptionParser(prepare.__doc__) p.set_outfile() opts, args = p.parse_args(args) outfile = opts.outfile if len(args) == 2: pairsfile, cdsfile = args pepfile = None elif len(args) == 3: pairsfile, cdsfile, pepfile = args else: sys.exit(not p.print_help()) f = Fasta(cdsfile) fp = open(pairsfile) fw = must_open(outfile, "w") if pepfile: assert outfile != "stdout", "Please specify outfile name." f2 = Fasta(pepfile) fw2 = must_open(outfile + ".pep", "w") for row in fp: if row[0] == '#': continue a, b = row.split()[:2] if a == b: logging.debug("Self pairs found: {0} - {1}. Ignored".format(a, b)) continue if a not in f: a = find_first_isoform(a, f) assert a, a if b not in f: b = find_first_isoform(b, f) assert b, b acds = f[a] bcds = f[b] SeqIO.write((acds, bcds), fw, "fasta") if pepfile: apep = f2[a] bpep = f2[b] SeqIO.write((apep, bpep), fw2, "fasta") fw.close() if pepfile: fw2.close()
def needle(args): """ %prog needle nw.pairs a.pep.fasta b.pep.fasta Take protein pairs and needle them Automatically writes output file `nw.scores` """ from jcvi.formats.fasta import Fasta, SeqIO p = OptionParser(needle.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) manager = mp.Manager() results = manager.list() needle_pool = mp.Pool(processes=mp.cpu_count()) pairsfile, apep, bpep = args afasta, bfasta = Fasta(apep), Fasta(bpep) fp = must_open(pairsfile) for i, row in enumerate(fp): a, b = row.split() a, b = afasta[a], bfasta[b] fa, fb = must_open("{0}_{1}_a.fasta".format(pairsfile, i), "w"), must_open( "{0}_{1}_b.fasta".format(pairsfile, i), "w") SeqIO.write([a], fa, "fasta") SeqIO.write([b], fb, "fasta") fa.close() fb.close() needlefile = "{0}_{1}_ab.needle".format(pairsfile, i) needle_pool.apply_async( _needle, (fa.name, fb.name, needlefile, a.id, b.id, results)) needle_pool.close() needle_pool.join() fp.close() scoresfile = "{0}.scores".format(pairsfile.rsplit(".")[0]) fw = must_open(scoresfile, "w") for result in results: print(result, file=fw) fw.close()
def prepare(args): """ %prog prepare pairsfile cdsfile > paired.cds.fasta Pick sequences from cdsfile to form pairs, ready to be calculated. The pairsfile can be generated from formats.blast.cscore(). The first two columns contain the pair. """ from jcvi.formats.fasta import Fasta, SeqIO p = OptionParser(prepare.__doc__) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) pairsfile, cdsfile = args f = Fasta(cdsfile) fp = open(pairsfile) fw = sys.stdout for row in fp: a, b = row.split()[:2] arec = f[a] brec = f[b] SeqIO.write((arec, brec), fw, "fasta")
def filter(args): """ %prog filter consensus.fasta Filter consensus sequence with min cluster size. """ from jcvi.formats.fasta import Fasta, SeqIO p = OptionParser(filter.__doc__) p.add_option("--minsize", default=10, type="int", help="Minimum cluster size") p.set_outfile() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args minsize = opts.minsize f = Fasta(fastafile, lazy=True) fw = must_open(opts.outfile, "w") for desc, rec in f.iterdescriptions_ordered(): if desc.startswith("singleton"): continue # consensus_for_cluster_0 with 63 sequences name, w, size, seqs = desc.split() assert w == "with" size = int(size) if size < minsize: continue SeqIO.write(rec, fw, "fasta")
def count(args): """ %prog count cdhit.consensus.fasta Scan the headers for the consensus clusters and count the number of reads. """ from jcvi.graphics.histogram import stem_leaf_plot from jcvi.utils.cbook import SummaryStats p = OptionParser(count.__doc__) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args f = Fasta(fastafile, lazy=True) sizes = [] for desc, rec in f.iterdescriptions_ordered(): if desc.startswith("singleton"): sizes.append(1) continue # consensus_for_cluster_0 with 63 sequences name, w, size, seqs = desc.split() assert w == "with" sizes.append(int(size)) s = SummaryStats(sizes) print >> sys.stderr, s stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def flip(args): """ %prog flip fastafile Go through each FASTA record, check against Genbank file and determines whether or not to flip the sequence. This is useful before updates of the sequences to make sure the same orientation is used. """ p = OptionParser(flip.__doc__) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args outfastafile = fastafile.rsplit(".", 1)[0] + ".flipped.fasta" fo = open(outfastafile, "w") f = Fasta(fastafile, lazy=True) for name, rec in f.iteritems_ordered(): tmpfasta = "a.fasta" fw = open(tmpfasta, "w") SeqIO.write([rec], fw, "fasta") fw.close() o = overlap([tmpfasta, name]) if o.orientation == '-': rec.seq = rec.seq.reverse_complement() SeqIO.write([rec], fo, "fasta") os.remove(tmpfasta)
def freebayes(args): """ %prog freebayes prefix ref.fa *.bam Call SNPs using freebayes. """ p = OptionParser(freebayes.__doc__) p.add_option("--mindepth", default=3, type="int", help="Minimum depth [default: %default]") p.add_option("--minqual", default=20, type="int", help="Minimum quality [default: %default]") opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) prefix, ref = args[0:2] bams = args[2:] cmd = "bamaddrg -R {0}" cmd += " " + " ".join("-b {0}".format(x) for x in bams) fmd = "freebayes --stdin -C {0} -f {1}".format(opts.mindepth, ref) seqids = list(Fasta(ref).iterkeys_ordered()) for s in seqids: outfile = prefix + ".{0}.vcf".format(s) print cmd.format(s), "|", fmd + " -r {0} -v {1}".format(s, outfile)
def score(args): """ %prog score blastfile query.fasta A.ids Add up the scores for each query seq. Go through the lines and for each query sequence, add up the scores when subject is in each pile by A.ids. """ from jcvi.formats.base import SetFile from jcvi.formats.fasta import Fasta p = OptionParser(score.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) blastfile, fastafile, idsfile = args ids = SetFile(idsfile) blast = Blast(blastfile) scores = defaultdict(int) for b in blast: query = b.query subject = b.subject if subject not in ids: continue scores[query] += b.score logging.debug("A total of {0} ids loaded.".format(len(ids))) f = Fasta(fastafile) for s in f.iterkeys_ordered(): sc = scores.get(s, 0) print "\t".join((s, str(sc)))
def digest(args): """ %prog digest fastafile NspI,BfuCI Digest fasta sequences to map restriction site positions. """ p = OptionParser(digest.__doc__) p.set_outfile() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) fastafile, enzymes = args enzymes = enzymes.split(",") enzymes = [x for x in AllEnzymes if str(x) in enzymes] f = Fasta(fastafile, lazy=True) fw = must_open(opts.outfile, "w") header = ["Contig", "Length"] + [str(x) for x in enzymes] print("\t".join(header), file=fw) for name, rec in f.iteritems_ordered(): row = [name, len(rec)] for e in enzymes: pos = e.search(rec.seq) pos = "na" if not pos else "|".join(str(x) for x in pos) row.append(pos) print("\t".join(str(x) for x in row), file=fw)
def dump(args): """ %prog dump fastafile Convert FASTA sequences to list of K-mers. """ p = OptionParser(dump.__doc__) p.add_option("-K", default=23, type="int", help="K-mer size [default: %default]") p.set_outfile() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args K = opts.K fw = must_open(opts.outfile, "w") f = Fasta(fastafile, lazy=True) for name, rec in f.iteritems_ordered(): kmers = list(make_kmers(rec.seq, K)) print >> fw, "\n".join(kmers) fw.close()
def fpkm(args): """ %prog fpkm fastafile *.bam Calculate FPKM values from BAM file. """ p = OptionParser(fpkm.__doc__) opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) fastafile = args[0] bamfiles = args[1:] # Create a DUMMY gff file for cuffdiff gffile = fastafile.rsplit(".", 1)[0] + ".gff" if need_update(fastafile, gffile): fw = open(gffile, "w") f = Fasta(fastafile, lazy=True) for key, size in f.itersizes_ordered(): print >> fw, "\t".join(str(x) for x in (key, "dummy", "transcript",\ 1, size, ".", ".", ".", "ID=" + key)) fw.close() logging.debug("Dummy GFF created: {0}".format(gffile)) cmd = "cuffdiff {0} {1}".format(gffile, " ".join(bamfiles)) sh(cmd)
def fragment(args): """ %prog fragment fastafile enzyme Cut the fastafile using the specified enzyme, and grab upstream and downstream nucleotide sequence along with the cut site. In this case, the sequences extracted are: |- PstI ============|=========== (-------) Sometimes we need to limit the size of the restriction fragments, for example the GBS protocol does not allow fragments larger than 800bp. |-PstI |- PstI |- PstI ~~~====|=============|==========~~~~~~~===|============ (---) (---) In this case, the second fragment is longer than 800bp, therefore the two ends are NOT extracted, as in the first fragment. """ p = OptionParser(fragment.__doc__) p.add_option( "--flank", default=150, type="int", help="Extract flanking bases of the cut sites", ) p.add_option( "--full", default=False, action="store_true", help="The full extraction mode", ) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) fastafile, enzyme = args flank = opts.flank assert flank > 0 extract = extract_full if opts.full else extract_ends tag = "full" if opts.full else "ends" assert enzyme in set(str(x) for x in AllEnzymes) fragfastafile = fastafile.split(".")[0] + ".{0}.flank{1}.{2}.fasta".format( enzyme, flank, tag ) enzyme = [x for x in AllEnzymes if str(x) == enzyme][0] f = Fasta(fastafile, lazy=True) fw = open(fragfastafile, "w") for name, rec in f.iteritems_ordered(): a = Analysis([enzyme], rec.seq) sites = a.full()[enzyme] extract(rec, sites, flank, fw) logging.debug("Fragments written to `{0}`.".format(fragfastafile))
def wgsim(args): """ %prog wgsim fastafile Run dwgsim on fastafile. """ p = OptionParser(wgsim.__doc__) p.add_option( "--erate", default=0.01, type="float", help="Base error rate of the read", ) p.add_option( "--noerrors", default=False, action="store_true", help="Simulate reads with no errors", ) p.add_option( "--genomesize", type="int", help="Genome size in Mb [default: estimate from data]", ) add_sim_options(p) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) (fastafile,) = args pf = op.basename(fastafile).split(".")[0] genomesize = opts.genomesize size = genomesize * 1000000 if genomesize else Fasta(fastafile).totalsize depth = opts.depth readlen = opts.readlen readnum = int(math.ceil(size * depth / (2 * readlen))) distance = opts.distance stdev = distance / 10 outpf = opts.outfile or "{0}.{1}bp.{2}x".format(pf, distance, depth) logging.debug("Total genome size: {0} bp".format(size)) logging.debug("Target depth: {0}x".format(depth)) logging.debug("Number of read pairs (2x{0}): {1}".format(readlen, readnum)) if opts.noerrors: opts.erate = 0 cmd = "dwgsim -e {0} -E {0}".format(opts.erate) if opts.noerrors: cmd += " -r 0 -R 0 -X 0 -y 0" cmd += " -d {0} -s {1}".format(distance, stdev) cmd += " -N {0} -1 {1} -2 {1}".format(readnum, readlen) cmd += " {0} {1}".format(fastafile, outpf) sh(cmd)
def build_all(self, componentfasta, targetfasta, newagp=None): f = Fasta(componentfasta, index=False) fw = open(targetfasta, "w") for ob, lines_with_same_ob in groupby(self, key=lambda x: x.object): lines = list(lines_with_same_ob) self.build_one(ob, lines, f, fw, newagp=newagp)
def n50(args): """ %prog n50 filename Given a file with a list of numbers denoting contig lengths, calculate N50. Input file can be both FASTA or a list of sizes. """ from jcvi.graphics.histogram import loghistogram p = OptionParser(n50.__doc__) p.add_option( "--print0", default=False, action="store_true", help="Print size and L50 to stdout", ) opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) ctgsizes = [] # Guess file format probe = open(args[0]).readline()[0] isFasta = probe == ">" if isFasta: for filename in args: f = Fasta(filename) ctgsizes += list(b for a, b in f.itersizes()) else: for row in must_open(args): try: ctgsize = int(float(row.split()[-1])) except ValueError: continue ctgsizes.append(ctgsize) a50, l50, nn50 = calculate_A50(ctgsizes) sumsize = sum(ctgsizes) minsize = min(ctgsizes) maxsize = max(ctgsizes) n = len(ctgsizes) print(", ".join(args), file=sys.stderr) summary = (sumsize, l50, nn50, minsize, maxsize, n) print( " ".join("{0}={1}".format(a, b) for a, b in zip(header, summary)), file=sys.stderr, ) loghistogram(ctgsizes) if opts.print0: print("\t".join(str(x) for x in (",".join(args), sumsize, l50))) return zip(header, summary)
def A50(args): """ %prog A50 contigs_A.fasta contigs_B.fasta ... Plots A50 graphics, see blog post (http://blog.malde.org/index.php/a50/) """ p = OptionParser(A50.__doc__) p.add_option("--overwrite", default=False, action="store_true", help="overwrite .rplot file if exists [default: %default]") p.add_option("--cutoff", default=0, type="int", dest="cutoff", help="use contigs above certain size [default: %default]") p.add_option("--stepsize", default=10, type="int", dest="stepsize", help="stepsize for the distribution [default: %default]") opts, args = p.parse_args(args) if not args: sys.exit(p.print_help()) import numpy as np from jcvi.utils.table import loadtable stepsize = opts.stepsize # use stepsize to speed up drawing rplot = "A50.rplot" if not op.exists(rplot) or opts.overwrite: fw = open(rplot, "w") header = "\t".join(("index", "cumsize", "fasta")) statsheader = ("Fasta", "L50", "N50", "Min", "Max", "Average", "Sum", "Counts") statsrows = [] print >>fw, header for fastafile in args: f = Fasta(fastafile, index=False) ctgsizes = [length for k, length in f.itersizes()] ctgsizes = np.array(ctgsizes) a50, l50, n50 = calculate_A50(ctgsizes, cutoff=opts.cutoff) cmin, cmax, cmean = min(ctgsizes), max(ctgsizes), np.mean(ctgsizes) csum, counts = np.sum(ctgsizes), len(ctgsizes) cmean = int(round(cmean)) statsrows.append((fastafile, l50, n50, cmin, cmax, cmean, csum, counts)) logging.debug("`{0}` ctgsizes: {1}".format(fastafile, ctgsizes)) tag = "{0} (L50={1})".format(\ op.basename(fastafile).rsplit(".", 1)[0], l50) logging.debug(tag) for i, s in zip(xrange(0, len(a50), stepsize), a50[::stepsize]): print >> fw, "\t".join((str(i), str(s / 1000000.), tag)) fw.close() table = loadtable(statsheader, statsrows) print >> sys.stderr, table generate_plot(rplot)
def summary(args): """ %prog summary gffile fastafile Print summary stats, including: - Gene/Exon/Intron - Number - Average size (bp) - Median size (bp) - Total length (Mb) - % of genome - % GC """ p = OptionParser(summary.__doc__) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) gff_file, ref = args s = Fasta(ref) g = make_index(gff_file) geneseqs, exonseqs, intronseqs = [], [], [] # Calc % GC for f in g.features_of_type("gene"): fid = f.id fseq = s.sequence({'chr': f.chrom, 'start': f.start, 'stop': f.stop}) geneseqs.append(fseq) exons = set((c.chrom, c.start, c.stop) for c in g.children(fid, 2) \ if c.featuretype == "exon") exons = list(exons) for chrom, start, stop in exons: fseq = s.sequence({'chr': chrom, 'start': start, 'stop': stop}) exonseqs.append(fseq) introns = range_interleave(exons) for chrom, start, stop in introns: fseq = s.sequence({'chr': chrom, 'start': start, 'stop': stop}) intronseqs.append(fseq) r = {} # Report for t, tseqs in zip(("Gene", "Exon", "Intron"), (geneseqs, exonseqs, intronseqs)): tsizes = [len(x) for x in tseqs] tsummary = SummaryStats(tsizes, dtype="int") r[t, "Number"] = tsummary.size r[t, "Average size (bp)"] = tsummary.mean r[t, "Median size (bp)"] = tsummary.median r[t, "Total length (Mb)"] = human_size(tsummary.sum, precision=0, target="Mb") r[t, "% of genome"] = percentage(tsummary.sum, s.totalsize, precision=0, mode=-1) r[t, "% GC"] = gc(tseqs) print >> sys.stderr, tabulate(r)
def minimap(args): """ %prog minimap ref.fasta query.fasta Wrap minimap2 aligner using query against sequences. When query and ref is the same, we are in "self-scan" mode (e.g. useful for finding internal duplications resulted from mis-assemblies). """ from jcvi.apps.grid import MakeManager from jcvi.formats.fasta import Fasta p = OptionParser(minimap.__doc__) p.add_option( "--chunks", type="int", default=2000000, help="Split ref.fasta into chunks of size in self-scan mode", ) p.set_outdir(outdir="outdir") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) ref, query = args chunks = opts.chunks outdir = opts.outdir if ref != query: raise NotImplementedError # "self-scan" mode # build faidx (otherwise, parallel make may complain) sh("samtools faidx {}".format(ref)) f = Fasta(ref) mkdir(outdir) mm = MakeManager() for name, size in f.itersizes(): start = 0 for end in range(chunks, size, chunks): fafile = op.join(outdir, "{}_{}_{}.fa".format(name, start + 1, end)) cmd = "samtools faidx {} {}:{}-{} -o {}".format( ref, name, start + 1, end, fafile) mm.add(ref, fafile, cmd) paffile = fafile.rsplit(".", 1)[0] + ".paf" cmd = "minimap2 -P {} {} > {}".format(fafile, fafile, paffile) mm.add(fafile, paffile, cmd) epsfile = fafile.rsplit(".", 1)[0] + ".eps" cmd = "minidot {} > {}".format(paffile, epsfile) mm.add(paffile, epsfile, cmd) start += chunks mm.write()
def needle(args): """ %prog needle pairs a.pep.fasta b.pep.fasta Take protein pairs and needle them. """ from Bio.Emboss.Applications import NeedleCommandline from jcvi.formats.fasta import Fasta, SeqIO from jcvi.formats.base import FileShredder p = OptionParser(needle.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) pairsfile, apep, bpep = args afasta = Fasta(apep) bfasta = Fasta(bpep) fp = open(pairsfile) for row in fp: fa = open(pairsfile + "_a.fasta", "w") fb = open(pairsfile + "_b.fasta", "w") a, b = row.split() a = afasta[a] b = bfasta[b] SeqIO.write([a], fa, "fasta") SeqIO.write([b], fb, "fasta") fa.close() fb.close() needlefile = pairsfile + "_ab.needle" needle_cline = NeedleCommandline(asequence=fa.name, bsequence=fb.name, gapopen=10, gapextend=0.5, outfile=needlefile) stdout, stderr = needle_cline() print >> sys.stderr, stdout + stderr #align = AlignIO.read(needlefile, "emboss") nh = NeedleHeader(needlefile) print "\t".join((a.id, b.id, nh.identity, nh.score)) FileShredder([fa.name, fb.name, needlefile])
def wgsim(args): """ %prog wgsim fastafile Run dwgsim on fastafile. """ p = OptionParser(wgsim.__doc__) p.add_option("--erate", default=.02, type="float", help="Base error rate of the read [default: %default]") p.add_option("--distance", default=500, type="int", help="Outer distance between the two ends [default: %default]") p.add_option("--genomesize", type="int", help="Genome size in Mb [default: estimate from data]") p.add_option("--readlen", default=100, type="int", help="Length of the read [default: %default]") p.add_option("--noerrors", default=False, action="store_true", help="Simulate reads with no errors [default: %default]") p.set_depth(depth=10) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args pf = fastafile.split(".")[0] genomesize = opts.genomesize size = genomesize * 1000000 if genomesize else Fasta(fastafile).totalsize depth = opts.depth readlen = opts.readlen readnum = size * depth / (2 * readlen) distance = opts.distance stdev = distance / 5 outpf = "{0}.{1}bp.{2}x".format(pf, distance, depth) distance -= 2 * readlen # Outer distance => Inner distance assert distance >= 0, "Outer distance must be >= 2 * readlen" logging.debug("Total genome size: {0} bp".format(size)) logging.debug("Target depth: {0}x".format(depth)) logging.debug("Number of read pairs (2x{0}): {1}".format(readlen, readnum)) if opts.noerrors: opts.erate = 0 cmd = "dwgsim -e {0} -E {0}".format(opts.erate) if opts.noerrors: cmd += " -r 0 -R 0 -X 0 -y 0" cmd += " -d {0} -s {1}".format(distance, stdev) cmd += " -N {0} -1 {1} -2 {1}".format(readnum, readlen) cmd += " {0} {1}".format(fastafile, outpf) sh(cmd)
def filter(args): """ %prog filter *.consensus.fasta Filter consensus sequence with min cluster size. """ from jcvi.formats.fasta import Fasta, SeqIO p = OptionParser(filter.__doc__) p.add_option("--minsize", default=2, type="int", help="Minimum cluster size") p.set_outfile() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) fastafiles = args minsize = opts.minsize totalreads = totalassembled = 0 fw = must_open(opts.outfile, "w") for i, fastafile in enumerate(fastafiles): f = Fasta(fastafile, lazy=True) pf = "s{0:03d}".format(i) nreads = nsingletons = nclusters = 0 for desc, rec in f.iterdescriptions_ordered(): nclusters += 1 if desc.startswith("singleton"): nsingletons += 1 nreads += 1 continue # consensus_for_cluster_0 with 63 sequences name, w, size, seqs = desc.split() assert w == "with" size = int(size) nreads += size if size < minsize: continue rec.description = rec.description.split(None, 1)[-1] rec.id = pf + "_" + rec.id SeqIO.write(rec, fw, "fasta") logging.debug("Scanned {0} clusters with {1} reads ..".format( nclusters, nreads)) cclusters, creads = nclusters - nsingletons, nreads - nsingletons logging.debug( "Saved {0} clusters (min={1}) with {2} reads (avg:{3}) [{4}]". format(cclusters, minsize, creads, creads / cclusters, pf)) totalreads += nreads totalassembled += nreads - nsingletons logging.debug("Total assembled: {0}".format( percentage(totalassembled, totalreads)))
def bes(args): """ %prog bes bacfasta clonename Use the clone name to download BES gss sequences from Genbank, map and then visualize. """ from jcvi.apps.align import run_blat p = OptionParser(bes.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bacfasta, clonename = args entrez([clonename, "--database=nucgss", "--skipcheck"]) besfasta = clonename + ".fasta" blatfile = clonename + ".bes.blat" run_blat( infile=besfasta, outfile=blatfile, db=bacfasta, pctid=95, hitlen=100, cpus=opts.cpus, ) aid, asize = next(Fasta(bacfasta).itersizes()) width = 50 msg = "=" * width msg += " " + aid print(msg, file=sys.stderr) ratio = width * 1.0 / asize _ = lambda x: int(round(x * ratio, 0)) blasts = [BlastLine(x) for x in open(blatfile)] for b in blasts: if b.orientation == "+": msg = " " * _(b.sstart) + "->" else: msg = " " * (_(b.sstop) - 2) + "<-" msg += " " * (width - len(msg) + 2) msg += b.query if b.orientation == "+": msg += " (hang={0})".format(b.sstart - 1) else: msg += " (hang={0})".format(asize - b.sstop) print(msg, file=sys.stderr)
def get_GC3(cdsfile): from jcvi.formats.fasta import Fasta f = Fasta(cdsfile, lazy=True) GC3 = {} for name, rec in f.iteritems_ordered(): positions = rec.seq[2::3].upper() gc_counts = sum(1 for x in positions if x in "GC") gc_ratio = gc_counts * 1. / len(positions) GC3[name] = gc_ratio return GC3
def merge(args): """ %prog merge gffiles Merge several gff files into one. When only one file is given, it is assumed to be a file with a list of gff files. """ p = OptionParser(merge.__doc__) set_outfile(p) opts, args = p.parse_args(args) nargs = len(args) if nargs < 1: sys.exit(not p.print_help()) if nargs == 1: listfile, = args fp = open(listfile) gffiles = [x.strip() for x in fp] else: gffiles = args outfile = opts.outfile deflines = set() fw = must_open(outfile, "w") fastarecs = {} for gffile in gffiles: fp = open(gffile) for row in fp: row = row.rstrip() if row[0] == '#': if row == FastaTag: break if row in deflines: continue else: deflines.add(row) print >> fw, row f = Fasta(gffile, lazy=True) for key, rec in f.iteritems_ordered(): if key in fastarecs.keys(): continue fastarecs[key] = rec print >> fw, FastaTag SeqIO.write(fastarecs.values(), fw, "fasta")
def count(args): """ %prog count fastafile jf.db Run dump - jellyfish - bin - bincount in serial. """ from bitarray import bitarray p = OptionParser(count.__doc__) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) fastafile, jfdb = args K = get_K(jfdb) cmd = "jellyfish query {0} -C | cut -d' ' -f 2".format(jfdb) t = must_open("tmp", "w") proc = Popen(cmd, stdin=PIPE, stdout=t) t.flush() f = Fasta(fastafile, lazy=True) for name, rec in f.iteritems_ordered(): kmers = list(make_kmers(rec.seq, K)) print("\n".join(kmers), file=proc.stdin) proc.stdin.close() logging.debug(cmd) proc.wait() a = bitarray() binfile = ".".join((fastafile, jfdb, "bin")) fw = open(binfile, "w") t.seek(0) for row in t: c = row.strip() a.append(int(c)) a.tofile(fw) logging.debug("Serialize {0} bits to `{1}`.".format(len(a), binfile)) fw.close() sh("rm {0}".format(t.name)) logging.debug( "Shared K-mers (K={0}) between `{1}` and `{2}` written to `{3}`.".format( K, fastafile, jfdb, binfile ) ) cntfile = ".".join((fastafile, jfdb, "cnt")) bincount([fastafile, binfile, "-o", cntfile, "-K {0}".format(K)]) logging.debug("Shared K-mer counts written to `{0}`.".format(cntfile))
def script(args): """ %prog script gffile cdna.fasta genome.fasta Parse gmap gff and produce script for sim4db to refine. """ p = OptionParser(script.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(p.print_help()) gffile, cdnafasta, genomefasta = args scriptfile = gffile + ".script" gff = Gff(gffile) fw = open(scriptfile, "w") cdnas = Fasta(cdnafasta, lazy=True) cdnas = dict((x, i) for (i, x) in enumerate(cdnas.iterkeys_ordered())) genomes = Fasta(genomefasta, lazy=True) genomes = dict((x, i) for (i, x) in enumerate(genomes.iterkeys_ordered())) extra = 50000 # 50-kb region surrounding the locus for g in gff: if g.type != "mRNA": continue cdna = g.attributes["Name"][0] genome = g.seqid ci = cdnas[cdna] gi = genomes[genome] strand = "-r" if g.strand == "-" else "-f" start, end = g.start, g.end start = max(0, start - extra) end += extra print >> fw, "{0} -e {1} -D {2} {3} {4}"\ .format(strand, ci, gi, start, end)
def __init__(self, filename, select=None): assert op.exists(filename), "File `{0}` not found".format(filename) # filename can be both .sizes file or FASTA formatted file sizesname = filename if not filename.endswith(".sizes"): sizesname = filename + ".sizes" filename = get_abs_path(filename) if need_update(filename, sizesname): cmd = "faSize" if which(cmd): cmd += " -detailed {0}".format(filename) sh(cmd, outfile=sizesname) else: from jcvi.formats.fasta import Fasta f = Fasta(filename) fw = open(sizesname, "w") for k, size in f.itersizes_ordered(): print("\t".join((k, str(size))), file=fw) fw.close() filename = sizesname assert filename.endswith(".sizes") super(Sizes, self).__init__(filename) self.fp = open(filename) self.filename = filename # get sizes for individual contigs, both in list and dict # this is to preserve the input order in the sizes file sizes = list(self.iter_sizes()) if select: assert select > 0 sizes = [x for x in sizes if x[1] >= select] self.sizes_mapping = dict(sizes) # get cumulative sizes, both in list and dict ctgs, sizes = zip(*sizes) self.sizes = sizes cumsizes = np.cumsum([0] + list(sizes)) self.ctgs = ctgs self.cumsizes = cumsizes self.cumsizes_mapping = dict(zip(ctgs, cumsizes))
def count(args): """ %prog count cdhit.consensus.fasta Scan the headers for the consensus clusters and count the number of reads. """ from jcvi.graphics.histogram import stem_leaf_plot from jcvi.utils.cbook import SummaryStats p = OptionParser(count.__doc__) p.add_option("--csv", help="Write depth per contig to file") opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) (fastafile, ) = args csv = open(opts.csv, "w") if opts.csv else None f = Fasta(fastafile, lazy=True) sizes = [] for desc, rec in f.iterdescriptions_ordered(): if desc.startswith("singleton"): sizes.append(1) continue # consensus_for_cluster_0 with 63 sequences if "with" in desc: name, w, size, seqs = desc.split() if csv: print("\t".join(str(x) for x in (name, size, len(rec))), file=csv) assert w == "with" sizes.append(int(size)) # MRD85:00603:02472;size=167; else: name, size, tail = desc.split(";") sizes.append(int(size.replace("size=", ""))) if csv: csv.close() logging.debug("File written to `%s`.", opts.csv) s = SummaryStats(sizes) print(s, file=sys.stderr) stem_leaf_plot(s.data, 0, 100, 20, title="Cluster size")
def n50(args): """ %prog n50 filename Given a file with a list of numbers denoting contig lengths, calculate N50. Input file can be both FASTA or a list of sizes. """ p = OptionParser(n50.__doc__) opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) ctgsizes = [] # Guess file format probe = open(args[0]).readline()[0] isFasta = (probe == '>') if isFasta: for filename in args: f = Fasta(filename) ctgsizes += list(b for a, b in f.itersizes()) else: for row in must_open(args): try: ctgsize = int(row.split()[-1]) except ValueError: continue ctgsizes.append(ctgsize) a50, l50, nn50 = calculate_A50(ctgsizes) sumsize = sum(ctgsizes) minsize = min(ctgsizes) maxsize = max(ctgsizes) n = len(ctgsizes) print >> sys.stderr, ", ".join(args) summary = (sumsize, l50, nn50, minsize, maxsize, n) print >> sys.stderr, " ".join("{0}={1}".format(a, b) for a, b in \ zip(header, summary)) loghistogram(ctgsizes, summary=False) return zip(header, summary)