def count(args): """ %prog count bamfile gtf Count the number of reads mapped using `htseq-count`. """ p = OptionParser(count.__doc__) sp1.add_argument("--type", default="exon", help="Only count feature type") p.set_cpus(cpus=8) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, gtf = args cpus = args.cpus pf = bamfile.split(".")[0] countfile = pf + ".count" if not need_update(bamfile, countfile): return nsorted = pf + "_nsorted" nsortedbam, nsortedsam = nsorted + ".bam", nsorted + ".sam" if need_update(bamfile, nsortedsam): cmd = "samtools sort -@ {0} -n {1} {2}".format(cpus, bamfile, nsorted) sh(cmd) cmd = "samtools view -@ {0} -h {1}".format(cpus, nsortedbam) sh(cmd, outfile=nsortedsam) if need_update(nsortedsam, countfile): cmd = "htseq-count --stranded=no --minaqual=10" cmd += " -t {0}".format(args.type) cmd += " {0} {1}".format(nsortedsam, gtf) sh(cmd, outfile=countfile)
def index(args): """ %prog index samfile/bamfile If SAM file, convert to BAM, sort and then index, using SAMTOOLS """ p = OptionParser(index.__doc__) sp1.add_argument("--fasta", dest="fasta", default=None, help="add @SQ header to the BAM file [default: %default]") sp1.add_argument("--unique", default=False, action="store_true", help="only retain uniquely mapped reads [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(p.print_help()) samfile, = args cpus = args.cpus fastafile = args.fasta if fastafile: assert op.exists(fastafile) bamfile = samfile.replace(".sam", ".bam") if fastafile: faifile = fastafile + ".fai" if need_update(fastafile, faifile): sh("samtools faidx {0}".format(fastafile)) cmd = "samtools view -bt {0} {1} -o {2}".\ format(faifile, samfile, bamfile) else: cmd = "samtools view -bS {0} -o {1}".\ format(samfile, bamfile) cmd += " -@ {0}".format(cpus) if args.unique: cmd += " -q 1" if samfile.endswith(".sam") and need_update(samfile, bamfile): sh(cmd) # Already sorted? if bamfile.endswith(".sorted.bam"): sortedbamfile = bamfile else: prefix = bamfile.replace(".bam", "") sortedbamfile = prefix + ".sorted.bam" if need_update(bamfile, sortedbamfile): cmd = "samtools sort {0} -o {1}".format(bamfile, sortedbamfile) cmd += " -@ {0}".format(cpus) sh(cmd) baifile = sortedbamfile + ".bai" if need_update(sortedbamfile, baifile): sh("samtools index {0}".format(sortedbamfile)) return sortedbamfile
def frompsl(args): """ %prog frompsl old.new.psl old.fasta new.fasta Generate chain file from psl file. The pipeline is describe in: <http://genomewiki.ucsc.edu/index.php/Minimal_Steps_For_LiftOver> """ from maize.formats.sizes import Sizes p = OptionParser(frompsl.__doc__) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) pslfile, oldfasta, newfasta = args pf = oldfasta.split(".")[0] # Chain together alignments from using axtChain chainfile = pf + ".chain" twobitfiles = [] for fastafile in (oldfasta, newfasta): tbfile = faToTwoBit(fastafile) twobitfiles.append(tbfile) oldtwobit, newtwobit = twobitfiles if need_update(pslfile, chainfile): cmd = "axtChain -linearGap=medium -psl {0}".format(pslfile) cmd += " {0} {1} {2}".format(oldtwobit, newtwobit, chainfile) sh(cmd) # Sort chain files sortedchain = chainfile.rsplit(".", 1)[0] + ".sorted.chain" if need_update(chainfile, sortedchain): cmd = "chainSort {0} {1}".format(chainfile, sortedchain) sh(cmd) # Make alignment nets from chains netfile = pf + ".net" oldsizes = Sizes(oldfasta).filename newsizes = Sizes(newfasta).filename if need_update((sortedchain, oldsizes, newsizes), netfile): cmd = "chainNet {0} {1} {2}".format(sortedchain, oldsizes, newsizes) cmd += " {0} /dev/null".format(netfile) sh(cmd) # Create liftOver chain file liftoverfile = pf + ".liftover.chain" if need_update((netfile, sortedchain), liftoverfile): cmd = "netChainSubset {0} {1} {2}".\ format(netfile, sortedchain, liftoverfile) sh(cmd)
def fpkm(args): """ %prog fpkm fastafile *.bam Calculate FPKM values from BAM file. """ p = OptionParser(fpkm.__doc__) opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) fastafile = args[0] bamfiles = args[1:] # Create a DUMMY gff file for cuffdiff gffile = fastafile.rsplit(".", 1)[0] + ".gff" if need_update(fastafile, gffile): fw = open(gffile, "w") f = Fasta(fastafile, lazy=True) for key, size in f.itersizes_ordered(): print >> fw, "\t".join(str(x) for x in (key, "dummy", "transcript",\ 1, size, ".", ".", ".", "ID=" + key)) fw.close() logging.debug("Dummy GFF created: {0}".format(gffile)) cmd = "cuffdiff {0} {1}".format(gffile, " ".join(bamfiles)) sh(cmd)
def first(args): """ %prog first N fastqfile(s) Get first N reads from file. """ from maize.apps.base import need_update p = OptionParser(first.__doc__) p.set_outfile() opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) N = int(args[0]) nlines = N * 4 fastqfiles = args[1:] fastqfile = fastqfiles[0] outfile = args.outfile if not need_update(fastqfiles, outfile): logging.debug("File `{0}` exists. Will not overwrite.".format(outfile)) return gz = fastqfile.endswith(".gz") for fastqfile in fastqfiles: if gz: cmd = "zcat {0} | head -n {1}".format(fastqfile, nlines) else: cmd = "head -n {0} {1}".format(nlines, fastqfile) sh(cmd, outfile=args.outfile, append=True)
def filter(args): """ %prog filter <deltafile|coordsfile> Produce a new delta/coords file and filter based on id% or cov%. Use `delta-filter` for .delta file. """ p = OptionParser(filter.__doc__) p.set_align(pctid=0, hitlen=0) sp1.add_argument("--overlap", default=False, action="store_true", help="Print overlap status (e.g. terminal, contained)") opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) pctid = args.pctid hitlen = args.hitlen filename, = args if pctid == 0 and hitlen == 0: return filename pf, suffix = filename.rsplit(".", 1) outfile = "".join((pf, ".P{0}L{1}.".format(int(pctid), int(hitlen)), suffix)) if not need_update(filename, outfile): return outfile if suffix == "delta": cmd = "delta-filter -i {0} -l {1} {2}".format(pctid, hitlen, filename) sh(cmd, outfile=outfile) return outfile fp = open(filename) fw = must_open(outfile, "w") for row in fp: try: c = CoordsLine(row) except AssertionError: continue if c.identity < pctid: continue if c.len2 < hitlen: continue if args.overlap and not c.overlap: continue outrow = row.rstrip() if args.overlap: ov = Overlap_types[c.overlap] outrow += "\t" + ov print >> fw, outrow return outfile
def __init__(self, filename, index=False): super(Maf, self).__init__(filename) indexfile = filename + ".idx" if index: if need_update(filename, indexfile): self.build_index(filename, indexfile) self.index = maf.Index(filename, indexfile) fp = open(filename) self.reader = maf.Reader(fp)
def __init__(self, filename, index=False): super(Maf, self).__init__(filename) indexfile = filename + ".idx" if index: if need_update(filename, indexfile): self.build_index(filename, indexfile) self.index = maf.Index(filename, indexfile) fp = open(filename) self.reader = maf.Reader(fp)
def fasta(args): """ %prog fasta fastqfiles Convert fastq to fasta and qual file. """ p = OptionParser(fasta.__doc__) sp1.add_argument("--seqtk", default=False, action="store_true", help="Use seqtk to convert") p.set_outdir() p.set_outfile(outfile=None) opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) fastqfiles = args outdir = args.outdir if outdir and outdir != ".": mkdir(outdir) fastqfile = fastqfiles[0] pf = op.basename(fastqfile) gzinput = pf.endswith(".gz") if gzinput: pf = pf.rsplit(".", 1)[0] pf, sf = pf.rsplit(".", 1) if sf not in ("fq", "fastq"): logging.debug("Assumed FASTA: suffix not `fq` or `fastq`") return fastqfile, None fastafile, qualfile = pf + ".fasta", pf + ".qual" outfile = args.outfile or fastafile outfile = op.join(outdir, outfile) if args.seqtk: if need_update(fastqfiles, outfile): for i, fastqfile in enumerate(fastqfiles): cmd = "seqtk seq -A {0} -L 30 -l 70".format(fastqfile) # First one creates file, following ones append to it sh(cmd, outfile=outfile, append=i) else: logging.debug("Outfile `{0}` already exists.".format(outfile)) return outfile, None for fastqfile in fastqfiles: SeqIO.convert(fastqfile, "fastq", fastafile, "fasta") SeqIO.convert(fastqfile, "fastq", qualfile, "qual") return fastafile, qualfile
def coverage(args): """ %prog coverage fastafile bamfile Calculate coverage for BAM file. BAM file will be sorted unless with --nosort. """ p = OptionParser(coverage.__doc__) sp1.add_argument("--format", default="bigwig", choices=("bedgraph", "bigwig", "coverage"), help="Output format") sp1.add_argument("--nosort", default=False, action="store_true", help="Do not sort BAM") p.set_outfile() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) fastafile, bamfile = args format = args.format if args.nosort: logging.debug("BAM sorting skipped") else: bamfile = index([bamfile, "--fasta={0}".format(fastafile)]) pf = bamfile.rsplit(".", 2)[0] sizesfile = Sizes(fastafile).filename cmd = "genomeCoverageBed -ibam {0} -g {1}".format(bamfile, sizesfile) if format in ("bedgraph", "bigwig"): cmd += " -bg" bedgraphfile = pf + ".bedgraph" sh(cmd, outfile=bedgraphfile) if format == "bedgraph": return bedgraphfile bigwigfile = pf + ".bigwig" cmd = "bedGraphToBigWig {0} {1} {2}".\ format(bedgraphfile, sizesfile, bigwigfile) sh(cmd) return bigwigfile coveragefile = pf + ".coverage" if need_update(fastafile, coveragefile): sh(cmd, outfile=coveragefile) gcf = GenomeCoverageFile(coveragefile) fw = must_open(args.outfile, "w") for seqid, cov in gcf.iter_coverage_seqid(): print >> fw, "\t".join((seqid, "{0:.1f}".format(cov))) fw.close()
def __init__(self, filename, select=None): assert op.exists(filename), "File `{0}` not found".format(filename) # filename can be both .sizes file or FASTA formatted file sizesname = filename if not filename.endswith(".sizes"): sizesname = filename + ".sizes" filename = get_abs_path(filename) if need_update(filename, sizesname): cmd = "faSize" if which(cmd): cmd += " -detailed {0}".format(filename) sh(cmd, outfile=sizesname) else: from jcvi.formats.fasta import Fasta f = Fasta(filename) fw = open(sizesname, "w") for k, size in f.itersizes_ordered(): fw.write("\t".join((k, str(size))) + "\n") fw.close() filename = sizesname assert filename.endswith(".sizes") super(Sizes, self).__init__(filename) self.fp = open(filename) self.filename = filename # get sizes for individual contigs, both in list and dict # this is to preserve the input order in the sizes file sizes = list(self.iter_sizes()) if select: assert select > 0 sizes = [x for x in sizes if x[1] >= select] self.sizes_mapping = dict(sizes) # get cumulative sizes, both in list and dict ctgs, sizes = zip(*sizes) self.sizes = sizes cumsizes = np.cumsum([0] + list(sizes)) self.ctgs = ctgs self.cumsizes = cumsizes self.cumsizes_mapping = dict(zip(ctgs, cumsizes))
def blast(args): """ %prog blast <deltafile|coordsfile> Covert delta or coordsfile to BLAST tabular output. """ p = OptionParser(blast.__doc__) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) deltafile, = args blastfile = deltafile.rsplit(".", 1)[0] + ".blast" if need_update(deltafile, blastfile): coords = Coords(deltafile) fw = open(blastfile, "w") for c in coords: print >> fw, c.blastline
def __init__(self, filename, sorted=False, header=False): if filename.endswith(".delta"): coordsfile = filename.rsplit(".", 1)[0] + ".coords" if need_update(filename, coordsfile): fromdelta([filename]) filename = coordsfile super(Coords, self).__init__(filename) fp = open(filename) if header: self.cmd = fp.next() for row in fp: try: self.append(CoordsLine(row)) except AssertionError: pass if sorted: self.ref_sort()
def pairs(args): """ See __doc__ for OptionParser.set_pairs(). """ import maize.formats.bed p = OptionParser(pairs.__doc__) p.set_pairs() opts, targs = p.parse_args(args) if len(targs) != 1: sys.exit(not p.print_help()) samfile, = targs bedfile = samfile.rsplit(".", 1)[0] + ".bed" if need_update(samfile, bedfile): cmd = "bamToBed -i {0}".format(samfile) sh(cmd, outfile=bedfile) args[args.index(samfile)] = bedfile return maize.formats.bed.pairs(args)
def wrapper(*args, **kwargs): assert outfile in kwargs, \ "You need to specify `outfile=` on function call" if infile in kwargs: infilename = listify(kwargs[infile]) for x in infilename: assert op.exists(x), \ "The specified infile `{0}` does not exist".format(x) outfilename = kwargs[outfile] if need_update(infilename, outfilename): return func(*args, **kwargs) else: msg = "File `{0}` exists. Computation skipped." \ .format(outfilename) logging.debug(msg) outfilename = listify(outfilename) for x in outfilename: assert op.exists(x), \ "Something went wrong, `{0}` not found".format(x) return outfilename
def faToTwoBit(fastafile): twobitfile = fastafile.rsplit(".", 1)[0] + ".2bit" cmd = "faToTwoBit {0} {1}".format(fastafile, twobitfile) if need_update(fastafile, twobitfile): sh(cmd) return twobitfile
def mstmap(args): """ %prog mstmap bcffile/vcffile > matrixfile Convert bcf/vcf format to mstmap input. """ from maize.assembly.geneticmap import MSTMatrix p = OptionParser(mstmap.__doc__) p.add_option("--dh", default=False, action="store_true", help="Double haploid population, no het [default: %default]") p.add_option("--freq", default=.2, type="float", help="Allele must be above frequency [default: %default]") p.add_option("--mindepth", default=3, type="int", help="Only trust genotype calls with depth [default: %default]") p.add_option("--missing_threshold", default=.25, type="float", help="Fraction missing must be below") p.add_option("--noheader", default=False, action="store_true", help="Do not print MSTmap run parameters [default: %default]") p.add_option("--pv4", default=False, action="store_true", help="Enable filtering strand-bias, tail distance bias, etc. " "[default: %default]") p.add_option("--freebayes", default=False, action="store_true", help="VCF output from freebayes") p.set_sep(sep=".", help="Use separator to simplify individual names") p.set_outfile() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) vcffile, = args if vcffile.endswith(".bcf"): bcffile = vcffile vcffile = bcffile.rsplit(".", 1)[0] + ".vcf" cmd = "bcftools view {0}".format(bcffile) cmd += " | vcfutils.pl varFilter" if not opts.pv4: cmd += " -1 0 -2 0 -3 0 -4 0 -e 0" if need_update(bcffile, vcffile): sh(cmd, outfile=vcffile) freq = opts.freq sep = opts.sep depth_index = 1 if opts.freebayes else 2 ptype = "DH" if opts.dh else "RIL6" nohet = ptype == "DH" fp = open(vcffile) genotypes = [] for row in fp: if row[:2] == "##": continue atoms = row.split() if row[0] == '#': ind = [x.split(sep)[0] for x in atoms[9:]] nind = len(ind) mh = ["locus_name"] + ind continue marker = "{0}.{1}".format(*atoms[:2]) geno = atoms[9:] geno = [encode_genotype(x, mindepth=opts.mindepth, depth_index=depth_index, nohet=nohet) for x in geno] assert len(geno) == nind f = 1. / nind if geno.count("A") * f < freq: continue if geno.count("B") * f < freq: continue if geno.count("-") * f > opts.missing_threshold: continue genotype = [marker] + geno genotypes.append(genotype) mm = MSTMatrix(genotypes, mh, ptype, opts.missing_threshold) mm.write(opts.outfile, header=(not opts.noheader))
def mstmap(args): """ %prog mstmap bcffile/vcffile > matrixfile Convert bcf/vcf format to mstmap input. """ from maize.assembly.geneticmap import MSTMatrix p = OptionParser(mstmap.__doc__) p.add_option("--dh", default=False, action="store_true", help="Double haploid population, no het [default: %default]") p.add_option("--freq", default=.2, type="float", help="Allele must be above frequency [default: %default]") p.add_option( "--mindepth", default=3, type="int", help="Only trust genotype calls with depth [default: %default]") p.add_option("--missing_threshold", default=.25, type="float", help="Fraction missing must be below") p.add_option("--noheader", default=False, action="store_true", help="Do not print MSTmap run parameters [default: %default]") p.add_option("--pv4", default=False, action="store_true", help="Enable filtering strand-bias, tail distance bias, etc. " "[default: %default]") p.add_option("--freebayes", default=False, action="store_true", help="VCF output from freebayes") p.set_sep(sep=".", help="Use separator to simplify individual names") p.set_outfile() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) vcffile, = args if vcffile.endswith(".bcf"): bcffile = vcffile vcffile = bcffile.rsplit(".", 1)[0] + ".vcf" cmd = "bcftools view {0}".format(bcffile) cmd += " | vcfutils.pl varFilter" if not opts.pv4: cmd += " -1 0 -2 0 -3 0 -4 0 -e 0" if need_update(bcffile, vcffile): sh(cmd, outfile=vcffile) freq = opts.freq sep = opts.sep depth_index = 1 if opts.freebayes else 2 ptype = "DH" if opts.dh else "RIL6" nohet = ptype == "DH" fp = open(vcffile) genotypes = [] for row in fp: if row[:2] == "##": continue atoms = row.split() if row[0] == '#': ind = [x.split(sep)[0] for x in atoms[9:]] nind = len(ind) mh = ["locus_name"] + ind continue marker = "{0}.{1}".format(*atoms[:2]) geno = atoms[9:] geno = [ encode_genotype(x, mindepth=opts.mindepth, depth_index=depth_index, nohet=nohet) for x in geno ] assert len(geno) == nind f = 1. / nind if geno.count("A") * f < freq: continue if geno.count("B") * f < freq: continue if geno.count("-") * f > opts.missing_threshold: continue genotype = [marker] + geno genotypes.append(genotype) mm = MSTMatrix(genotypes, mh, ptype, opts.missing_threshold) mm.write(opts.outfile, header=(not opts.noheader))