def blast(args): """ %prog blast ref.fasta query.fasta Calls blast and then filter the BLAST hits. Default is megablast. """ task_choices = ("blastn", "blastn-short", "dc-megablast", \ "megablast", "vecscreen") p = OptionParser(blast.__doc__) p.set_align(pctid=None, evalue=.01) p.add_option("--wordsize", type="int", help="Word size [default: %default]") p.add_option("--best", default=1, type="int", help="Only look for best N hits [default: %default]") p.add_option("--task", default="megablast", choices=task_choices, help="Task of the blastn [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) reffasta, queryfasta = args q = op.basename(queryfasta).split(".")[0] r = op.basename(reffasta).split(".")[0] blastfile = "{0}.{1}.blast".format(q, r) run_megablast(infile=queryfasta, outfile=blastfile, db=reffasta, wordsize=opts.wordsize, pctid=opts.pctid, evalue=opts.evalue, hitlen=None, best=opts.best, task=opts.task, cpus=opts.cpus) return blastfile
def batchoverlap(args): """ %prog batchoverlap pairs.txt outdir Check overlaps between pairs of sequences. """ p = OptionParser(batchoverlap.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) pairsfile, outdir = args fp = open(pairsfile) cmds = [] mkdir("overlaps") for row in fp: a, b = row.split()[:2] oa = op.join(outdir, a + ".fa") ob = op.join(outdir, b + ".fa") cmd = "python -m jcvi.assembly.goldenpath overlap {0} {1}".format(oa, ob) cmd += " -o overlaps/{0}_{1}.ov".format(a, b) cmds.append(cmd) print "\n".join(cmds)
def filtervcf(args): """ %prog filtervcf NA12878.hg38.vcf.gz Filter lobSTR VCF using script shipped in lobSTR. Input file can be a list of vcf files. """ p = OptionParser(filtervcf.__doc__) p.set_home("lobstr", default="/mnt/software/lobSTR") p.set_aws_opts(store="hli-mv-data-science/htang/str") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) samples, = args lhome = opts.lobstr_home store = opts.output_path if samples.endswith((".vcf", ".vcf.gz")): vcffiles = [samples] else: vcffiles = [x.strip() for x in must_open(samples)] vcffiles = [x for x in vcffiles if ".filtered." not in x] run_args = [(x, lhome, x.startswith("s3://") and store) for x in vcffiles] cpus = min(opts.cpus, len(run_args)) p = Pool(processes=cpus) for res in p.map_async(run_filter, run_args).get(): continue
def cufflinks(args): """ %prog cufflinks folder reference Run cufflinks on a folder containing tophat results. """ p = OptionParser(cufflinks.__doc__) p.add_option("--gtf", help="Reference annotation [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) folder, reference = args os.chdir(folder) bams = glob("*tophat/accepted_hits.bam") for bam in bams: pf, ab = op.split(bam) outdir = op.join(pf, "cufflinks") if op.exists(outdir): logging.debug("Directory {0} found. Skipping.".format(outdir)) continue cmd = "cufflinks" cmd += " -o {0}".format(outdir) cmd += " -p {0}".format(opts.cpus) if opts.gtf: cmd += " -g {0}".format(opts.gtf) cmd += " --frag-bias-correct {0}".format(reference) cmd += " --multi-read-correct" cmd += " {0}".format(bam) sh(cmd)
def blat(args): """ %prog blat ref.fasta query.fasta Calls blat and filters BLAST hits. """ p = OptionParser(blat.__doc__) p.set_align(pctid=95, hitlen=30) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) reffasta, queryfasta = args blastfile = get_outfile(reffasta, queryfasta, suffix="blat") run_blat( infile=queryfasta, outfile=blastfile, db=reffasta, pctid=opts.pctid, hitlen=opts.hitlen, cpus=opts.cpus, overwrite=False, ) return blastfile
def filterdata(args): """ %prog filterdata data.bin samples.ids STR.ids allele_freq remove.ids final.ids Filter subset of data after dropping remove.ids. """ p = OptionParser(filterdata.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 6: sys.exit(not p.print_help()) binfile, sampleids, strids, af, remove, final = args df, m, samples, loci = read_binfile(binfile, sampleids, strids) remove = [x.strip() for x in open(remove)] removes = set(remove) final = [x.strip() for x in open(final)] assert len(loci) == len(remove) + len(final) fp = open(af) percentiles = {} for row in fp: sname, counts = row.split() countsd = af_to_counts(counts) percentile = counts_to_percentile(countsd) percentiles[sname] = percentile run_args = [] for i, sname in enumerate(loci): if sname in removes: continue a = m[:, i] percentile = percentiles[sname] run_args.append((i, a, percentile)) cpus = min(opts.cpus, len(run_args)) p = Pool(processes=cpus) res = [] for r in p.map_async(convert_to_percentile, run_args).get(): res.append(r) res.sort() # Write mask (P-value) matrix ii, pvalues = zip(*res) m = np.vstack(pvalues).T write_csv("final.mask.tsv", m, samples, final) df.drop(remove, inplace=True, axis=1) df.columns = final # Save a copy of the raw numpy array filtered_bin = "filtered.bin" m = df.as_matrix() m[m < 0] = -1 m.tofile(filtered_bin) logging.debug("Binary matrix written to `{}`".format(filtered_bin)) # Write data output df.to_csv("final.data.tsv", sep="\t", index_label="SampleKey")
def deduplicate(args): """ %prog deduplicate fastafile Wraps `cd-hit-est` to remove duplicate sequences. """ p = OptionParser(deduplicate.__doc__) p.set_align(pctid=98) p.add_option("--reads", default=False, action="store_true", help="Use `cd-hit-454` to deduplicate [default: %default]") p.add_option("--samestrand", default=False, action="store_true", help="Enforce same strand alignment [%default: %default]") p.set_home("cdhit") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args identity = opts.pctid / 100. cmd = "cd-hit-454" if opts.reads else "cd-hit-est" cmd = op.join(opts.cdhit_home, cmd) cmd += " -c {0}".format(identity) cmd += " -d 0" # include complete defline if opts.samestrand: cmd += " -r 0" cmd += " -M 0 -T {0} -i {1} -o {1}.cdhit".format(opts.cpus, fastafile) sh(cmd) dd = fastafile + ".cdhit" return dd
def count(args): """ %prog count bamfile gtf Count the number of reads mapped using `htseq-count`. """ p = OptionParser(count.__doc__) p.add_option("--type", default="exon", help="Only count feature type") p.set_cpus(cpus=8) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, gtf = args cpus = opts.cpus pf = bamfile.split(".")[0] countfile = pf + ".count" if not need_update(bamfile, countfile): return nsorted = pf + "_nsorted" nsortedbam, nsortedsam = nsorted + ".bam", nsorted + ".sam" if need_update(bamfile, nsortedsam): cmd = "samtools sort -@ {0} -n {1} {2}".format(cpus, bamfile, nsorted) sh(cmd) cmd = "samtools view -@ {0} -h {1}".format(cpus, nsortedbam) sh(cmd, outfile=nsortedsam) if need_update(nsortedsam, countfile): cmd = "htseq-count --stranded=no --minaqual=10" cmd += " -t {0}".format(opts.type) cmd += " {0} {1}".format(nsortedsam, gtf) sh(cmd, outfile=countfile)
def alignextend(args): """ %prog alignextend ref.fasta read.1.fastq read.2.fastq Wrapper around AMOS alignextend. """ p = OptionParser(alignextend.__doc__) p.add_option("--nosuffix", default=False, action="store_true", help="Do not add /1/2 suffix to the read [default: %default]") p.set_home("amos") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) ref, r1, r2 = args pf = op.basename(r1).split(".")[0] cmd = op.join(opts.amos_home, "src/Experimental/alignextend.pl") if not opts.nosuffix: cmd += " -suffix" bwa_idx = "{0}.ref.fa.sa".format(pf) if not need_update(ref, bwa_idx): cmd += " -noindex" cmd += " -threads {0}".format(opts.cpus) offset = guessoffset([r1]) if offset == 64: cmd += " -I" cmd += " ".join(("", pf, ref, r1, r2)) sh(cmd)
def align(args): """ %prog align database.fasta read1.fq read2.fq Wrapper for `gsnap` single-end or paired-end, depending on the number of args. """ from jcvi.formats.fasta import join from jcvi.formats.fastq import guessoffset from jcvi.projects.tgbs import snp p = OptionParser(align.__doc__) p.add_option("--join", default=False, action="store_true", help="Join sequences with padded 50Ns") p.add_option("--rnaseq", default=False, action="store_true", help="Input is RNA-seq reads, turn splicing on") p.add_option("--snp", default=False, action="store_true", help="Call SNPs after GSNAP") p.set_cpus() opts, args = p.parse_args(args) if len(args) == 2: logging.debug("Single-end alignment") elif len(args) == 3: logging.debug("Paired-end alignment") else: sys.exit(not p.print_help()) dbfile, readfile = args[0:2] if opts.join: dbfile = join([dbfile, "--gapsize=50", "--newid=chr1"]) assert op.exists(dbfile) and op.exists(readfile) prefix = get_prefix(readfile, dbfile) logfile = prefix + ".log" gsnapfile = prefix + ".gsnap" if not need_update((dbfile, readfile), gsnapfile): logging.error("`{0}` exists. `gsnap` already run.".format(gsnapfile)) else: dbdir, dbname = check_index(dbfile) cmd = "gsnap -D {0} -d {1}".format(dbdir, dbname) cmd += " -B 5 -m 0.1 -i 2 -n 3" # memory, mismatch, indel penalty, nhits if opts.rnaseq: cmd += " -N 1" cmd += " -t {0}".format(opts.cpus) cmd += " --gmap-mode none --nofails" if readfile.endswith(".gz"): cmd += " --gunzip" try: offset = "sanger" if guessoffset([readfile]) == 33 else "illumina" cmd += " --quality-protocol {0}".format(offset) except AssertionError: pass cmd += " " + " ".join(args[1:]) sh(cmd, outfile=gsnapfile, errfile=logfile) if opts.snp: snp([gsnapfile, "--cpus={0}".format(opts.cpus)]) return gsnapfile, logfile
def extract(args): """ %prog extract bamfile contig Extract sub-bam for just one contig. """ p = OptionParser(extract.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, contig = args cpus = opts.cpus pf = bamfile.split(".")[0] outfile = ".".join((contig.split("|")[0], pf, "bam")) if op.exists(outfile): logging.error("Output name exists: `{}`".format(outfile)) return if need_update(bamfile, outfile): cmd = 'samtools view {} "{}" -@ {}'.format(bamfile, contig, cpus) cmd += " -b -o {}".format(outfile) sh(cmd) index([outfile, "--cpus={}".format(cpus)])
def cib(args): """ %prog cib bamfile samplekey Convert BAM to CIB (a binary storage of int8 per base). """ p = OptionParser(cib.__doc__) p.add_option("--prefix", help="Report seqids with this prefix only") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, samplekey = args mkdir(samplekey) bam = pysam.AlignmentFile(bamfile, "rb") refs = [x for x in bam.header["SQ"]] prefix = opts.prefix if prefix: refs = [x for x in refs if x["SN"].startswith(prefix)] task_args = [] for r in refs: task_args.append((bamfile, r, samplekey)) cpus = min(opts.cpus, len(task_args)) logging.debug("Use {} cpus".format(cpus)) p = Pool(processes=cpus) for res in p.imap(bam_to_cib, task_args): continue
def gcn(args): """ %prog gcn gencode.v26.exonunion.bed data/*.vcf.gz Compile gene copy njumber based on CANVAS results. """ p = OptionParser(gcn.__doc__) p.set_cpus() p.set_tmpdir(tmpdir="tmp") p.set_outfile() opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) exonbed = args[0] canvasvcfs = args[1:] tsvfile = opts.outfile tmpdir = opts.tmpdir mkdir(tmpdir) set_tempdir(tmpdir) df = vcf_to_df(canvasvcfs, exonbed, opts.cpus) for suffix in (".avgcn", ".medcn"): df_to_tsv(df, tsvfile, suffix)
def blat(args): """ %prog blat old.fasta new.fasta Generate psl file using blat. """ p = OptionParser(blat.__doc__) p.add_option("--minscore", default=100, type="int", help="Matches minus mismatches gap penalty [default: %default]") p.add_option("--minid", default=98, type="int", help="Minimum sequence identity [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) oldfasta, newfasta = args twobitfiles = [] for fastafile in args: tbfile = faToTwoBit(fastafile) twobitfiles.append(tbfile) oldtwobit, newtwobit = twobitfiles cmd = "pblat -threads={0}".format(opts.cpus) if which("pblat") else "blat" cmd += " {0} {1}".format(oldtwobit, newfasta) cmd += " -tileSize=12 -minScore={0} -minIdentity={1} ".\ format(opts.minscore, opts.minid) pslfile = "{0}.{1}.psl".format(*(op.basename(x).split('.')[0] \ for x in (newfasta, oldfasta))) cmd += pslfile sh(cmd)
def bam(args): """ %prog snp input.gsnap ref.fasta Convert GSNAP output to BAM. """ from jcvi.formats.sizes import Sizes from jcvi.formats.sam import index p = OptionParser(bam.__doc__) p.set_home("eddyyeh") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) gsnapfile, fastafile = args EYHOME = opts.eddyyeh_home pf = gsnapfile.rsplit(".", 1)[0] uniqsam = pf + ".unique.sam" if need_update((gsnapfile, fastafile), uniqsam): cmd = op.join(EYHOME, "gsnap2gff3.pl") sizesfile = Sizes(fastafile).filename cmd += " --format sam -i {0} -o {1}".format(gsnapfile, uniqsam) cmd += " -u -l {0} -p {1}".format(sizesfile, opts.cpus) sh(cmd) index([uniqsam])
def snp(args): """ %prog snp input.gsnap Run SNP calling on GSNAP output after apps.gsnap.align(). """ p = OptionParser(snp.__doc__) p.set_home("eddyyeh") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) gsnapfile, = args EYHOME = opts.eddyyeh_home pf = gsnapfile.rsplit(".", 1)[0] nativefile = pf + ".native" if need_update(gsnapfile, nativefile): cmd = op.join(EYHOME, "convert2native.pl") cmd += " --gsnap {0} -o {1}".format(gsnapfile, nativefile) cmd += " -proc {0}".format(opts.cpus) sh(cmd) snpfile = pf + ".snp" if need_update(nativefile, snpfile): cmd = op.join(EYHOME, "SNPs/SNP_Discovery-short.pl") cmd += " --native {0} -o {1}".format(nativefile, snpfile) cmd += " -a 2 -ac 0.3 -c 0.8" sh(cmd)
def density(args): """ %prog density test.clm Estimate link density of contigs. """ p = OptionParser(density.__doc__) p.add_option("--save", default=False, action="store_true", help="Write log densitites of contigs to file") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) clmfile, = args clm = CLMFile(clmfile) pf = clmfile.rsplit(".", 1)[0] if opts.save: logdensities = clm.calculate_densities() densityfile = pf + ".density" fw = open(densityfile, "w") for name, logd in logdensities.items(): s = clm.tig_to_size[name] print >> fw, "\t".join(str(x) for x in (name, s, logd)) fw.close() logging.debug("Density written to `{}`".format(densityfile)) tourfile = clmfile.rsplit(".", 1)[0] + ".tour" tour = clm.activate(tourfile=tourfile, backuptour=False) clm.flip_all(tour) clm.flip_whole(tour) clm.flip_one(tour)
def beagle(args): """ %prog beagle input.vcf 1 Use BEAGLE4.1 to impute vcf on chromosome 1. """ p = OptionParser(beagle.__doc__) p.set_home("beagle") p.set_ref() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) vcffile, chr = args pf = vcffile.rsplit(".", 1)[0] outpf = pf + ".beagle" outfile = outpf + ".vcf.gz" mm = MakeManager() beagle_cmd = opts.beagle_home kg = op.join(opts.ref, "1000GP_Phase3") cmd = beagle_cmd + " gt={0}".format(vcffile) cmd += " ref={0}/chr{1}.1kg.phase3.v5a.bref".format(kg, chr) cmd += " map={0}/plink.chr{1}.GRCh37.map".format(kg, chr) cmd += " out={0}".format(outpf) cmd += " nthreads=16 gprobs=true" mm.add(vcffile, outfile, cmd) mm.write()
def jellyfish(args): """ %prog jellyfish [*.fastq|*.fasta] Run jellyfish to dump histogram to be used in kmer.histogram(). """ from jcvi.apps.base import getfilesize from jcvi.utils.cbook import human_size p = OptionParser(jellyfish.__doc__) p.add_option("-K", default=23, type="int", help="K-mer size [default: %default]") p.add_option("--coverage", default=40, type="int", help="Expected sequence coverage [default: %default]") p.add_option("--prefix", default="jf", help="Database prefix [default: %default]") p.add_option("--nohist", default=False, action="store_true", help="Do not print histogram [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) fastqfiles = args K = opts.K coverage = opts.coverage totalfilesize = sum(getfilesize(x) for x in fastqfiles) fq = fastqfiles[0] pf = opts.prefix gzip = fq.endswith(".gz") hashsize = totalfilesize / coverage logging.debug("Total file size: {0}, hashsize (-s): {1}".\ format(human_size(totalfilesize, a_kilobyte_is_1024_bytes=True), hashsize)) jfpf = "{0}-K{1}".format(pf, K) jfdb = jfpf fastqfiles = " ".join(fastqfiles) cmd = "jellyfish count -t {0} -C -o {1}".format(opts.cpus, jfpf) cmd += " -s {0} -m {1}".format(hashsize, K) if gzip: cmd = "gzip -dc {0} | ".format(fastqfiles) + cmd + " /dev/fd/0" else: cmd += " " + fastqfiles if need_update(fastqfiles, jfdb): sh(cmd) if opts.nohist: return jfhisto = jfpf + ".histogram" cmd = "jellyfish histo -t 64 {0} -o {1}".format(jfdb, jfhisto) if need_update(jfdb, jfhisto): sh(cmd)
def impute(args): """ %prog impute input.vcf hs37d5.fa 1 Use IMPUTE2 to impute vcf on chromosome 1. """ from pyfaidx import Fasta p = OptionParser(impute.__doc__) p.set_home("shapeit") p.set_home("impute") p.set_ref() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) vcffile, fastafile, chr = args mm = MakeManager() pf = vcffile.rsplit(".", 1)[0] hapsfile = pf + ".haps" kg = op.join(opts.ref, "1000GP_Phase3") shapeit_phasing(mm, chr, vcffile, opts) fasta = Fasta(fastafile) size = len(fasta[chr]) binsize = 5000000 bins = size / binsize # 5Mb bins if size % binsize: bins += 1 impute_cmd = op.join(opts.impute_home, "impute2") chunks = [] for x in xrange(bins + 1): chunk_start = x * binsize + 1 chunk_end = min(chunk_start + binsize - 1, size) outfile = pf + ".chunk{0:02d}.impute2".format(x) mapfile = "{0}/genetic_map_chr{1}_combined_b37.txt".format(kg, chr) rpf = "{0}/1000GP_Phase3_chr{1}".format(kg, chr) cmd = impute_cmd + " -m {0}".format(mapfile) cmd += " -known_haps_g {0}".format(hapsfile) cmd += " -h {0}.hap.gz -l {0}.legend.gz".format(rpf) cmd += " -Ne 20000 -int {0} {1}".format(chunk_start, chunk_end) cmd += " -o {0} -allow_large_regions -seed 367946".format(outfile) cmd += " && touch {0}".format(outfile) mm.add(hapsfile, outfile, cmd) chunks.append(outfile) # Combine all the files imputefile = pf + ".impute2" cmd = "cat {0} > {1}".format(" ".join(chunks), imputefile) mm.add(chunks, imputefile, cmd) # Convert to vcf vcffile = pf + ".impute2.vcf" cmd = "python -m jcvi.formats.vcf fromimpute2 {0} {1} {2} > {3}".\ format(imputefile, fastafile, chr, vcffile) mm.add(imputefile, vcffile, cmd) mm.write()
def last(args): """ %prog database.fasta query.fasta Run LAST by calling LASTDB and LASTAL. LAST program available: <http://last.cbrc.jp> Works with LAST-719. """ p = OptionParser(last.__doc__) p.add_option("--path", help="Specify LAST path") p.add_option("--mask", default=False, action="store_true", help="Invoke -c in lastdb") p.add_option("--format", default="BlastTab", choices=("TAB", "MAF", "BlastTab", "BlastTab+"), help="Output format") p.add_option("--minlen", default=0, type="int", help="Filter alignments by how many bases match") p.add_option("--minid", default=0, type="int", help="Minimum sequence identity") p.set_cpus() p.set_params() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) subject, query = args path = opts.path cpus = opts.cpus getpath = lambda x: op.join(path, x) if path else x lastdb_bin = getpath("lastdb") lastal_bin = getpath("lastal") subjectdb = subject.rsplit(".", 1)[0] run_lastdb(infile=subject, outfile=subjectdb + ".prj", mask=opts.mask, \ lastdb_bin=lastdb_bin) u = 2 if opts.mask else 0 cmd = "{0} -u {1}".format(lastal_bin, u) cmd += " -P {0} -i3G".format(cpus) cmd += " -f {0}".format(opts.format) cmd += " {0} {1}".format(subjectdb, query) minlen = opts.minlen minid = opts.minid extra = opts.extra assert minid != 100, "Perfect match not yet supported" mm = minid / (100 - minid) if minlen: extra += " -e{0}".format(minlen) if minid: extra += " -r1 -q{0} -a{0} -b{0}".format(mm) if extra: cmd += " " + extra.strip() lastfile = get_outfile(subject, query, suffix="last") sh(cmd, outfile=lastfile)
def index(args): """ %prog index samfile/bamfile If SAM file, convert to BAM, sort and then index, using SAMTOOLS """ p = OptionParser(index.__doc__) p.add_option("--fasta", dest="fasta", default=None, help="add @SQ header to the BAM file [default: %default]") p.add_option("--unique", default=False, action="store_true", help="only retain uniquely mapped reads [default: %default]") p.set_cpus(cpus=8) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(p.print_help()) samfile, = args cpus = opts.cpus fastafile = opts.fasta if fastafile: assert op.exists(fastafile) bamfile = samfile.replace(".sam", ".bam") if fastafile: faifile = fastafile + ".fai" if need_update(fastafile, faifile): sh("samtools faidx {0}".format(fastafile)) cmd = "samtools view -bt {0} {1} -F 4 -o {2}".\ format(faifile, samfile, bamfile) else: cmd = "samtools view -bS {0} -F 4 -o {1}".\ format(samfile, bamfile) cmd += " -@ {0}".format(cpus) if opts.unique: cmd += " -q 1" if samfile.endswith(".sam") and need_update(samfile, bamfile): sh(cmd) # Already sorted? if bamfile.endswith(".sorted.bam"): sortedbamfile = bamfile else: prefix = bamfile.replace(".bam", "") sortedbamfile = prefix + ".sorted.bam" if need_update(bamfile, sortedbamfile): cmd = "samtools sort {0} {1}.sorted".format(bamfile, prefix) cmd += " -@ {0}".format(cpus) sh(cmd) baifile = sortedbamfile + ".bai" if need_update(sortedbamfile, baifile): sh("samtools index {0}".format(sortedbamfile)) return sortedbamfile
def mito(args): """ %prog mito chrM.fa input.bam Identify mitochondrial deletions. """ p = OptionParser(mito.__doc__) p.set_aws_opts(store="hli-mv-data-science/htang/mito-deletions") p.add_option("--realignonly", default=False, action="store_true", help="Realign only") p.add_option("--svonly", default=False, action="store_true", help="Run Realign => SV calls only") p.add_option("--support", default=1, type="int", help="Minimum number of supporting reads") p.set_home("speedseq", default="/mnt/software/speedseq/bin") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) chrMfa, bamfile = args store = opts.output_path cleanup = not opts.nocleanup if not op.exists(chrMfa): logging.debug("File `{}` missing. Exiting.".format(chrMfa)) return chrMfai = chrMfa + ".fai" if not op.exists(chrMfai): cmd = "samtools index {}".format(chrMfa) sh(cmd) if not bamfile.endswith(".bam"): bamfiles = [x.strip() for x in open(bamfile)] else: bamfiles = [bamfile] if store: computed = ls_s3(store) computed = [op.basename(x).split('.')[0] for x in computed if \ x.endswith(".depth")] remaining_samples = [x for x in bamfiles \ if op.basename(x).split(".")[0] not in computed] logging.debug("Already computed on `{}`: {}".\ format(store, len(bamfiles) - len(remaining_samples))) bamfiles = remaining_samples logging.debug("Total samples: {}".format(len(bamfiles))) for bamfile in bamfiles: run_mito(chrMfa, bamfile, opts, realignonly=opts.realignonly, svonly=opts.svonly, store=store, cleanup=cleanup)
def tophat(args): """ %prog tophat folder reference Run tophat on a folder of reads. """ from jcvi.apps.bowtie import check_index from jcvi.formats.fastq import guessoffset p = OptionParser(tophat.__doc__) p.add_option("--gtf", help="Reference annotation [default: %default]") p.add_option("--single", default=False, action="store_true", help="Single end mapping") p.add_option("--intron", default=15000, type="int", help="Max intron size [default: %default]") p.add_option("--dist", default=-50, type="int", help="Mate inner distance [default: %default]") p.add_option("--stdev", default=50, type="int", help="Mate standard deviation [default: %default]") p.set_phred() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) num = 1 if opts.single else 2 folder, reference = args reference = check_index(reference) for p, prefix in iter_project(folder, n=num): outdir = "{0}_tophat".format(prefix) outfile = op.join(outdir, "accepted_hits.bam") if op.exists(outfile): logging.debug("File `{0}` found. Skipping.".format(outfile)) continue cmd = "tophat -p {0}".format(opts.cpus) if opts.gtf: cmd += " -G {0}".format(opts.gtf) cmd += " -o {0}".format(outdir) if num == 1: # Single-end a, = p else: # Paired-end a, b = p cmd += " --max-intron-length {0}".format(opts.intron) cmd += " --mate-inner-dist {0}".format(opts.dist) cmd += " --mate-std-dev {0}".format(opts.stdev) phred = opts.phred or str(guessoffset([a])) if phred == "64": cmd += " --phred64-quals" cmd += " {0} {1}".format(reference, " ".join(p)) sh(cmd)
def prepare(args): """ %prog prepare genomesize *.fastq Prepare MERACULOUS configuation file. Genome size should be entered in Mb. """ p = OptionParser(prepare.__doc__ + FastqNamings) p.add_option("-K", default=51, type="int", help="K-mer size") p.set_cpus(cpus=32) opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) genomesize = float(args[0]) / 1000 fnames = args[1:] for x in fnames: assert op.exists(x), "File `{0}` not found.".format(x) s = comment_banner("Meraculous params file") + "\n" s += comment_banner("Basic parameters") + "\n" s += "# Describe the libraries ( one line per library )\n" s += "# " + " ".join(header.split()) + "\n" libs = get_libs(fnames) lib_seqs = [] rank = 0 for lib, fs in libs: size = lib.size if size == 0: continue rank += 1 library_name = lib.library_name name = library_name.replace("-", "") wildcard = "{0}*.1.*,{0}*.2.*".format(library_name) rl = max(readlen([x]) for x in fs) lib_seq = lib.get_lib_seq(wildcard, name, rl, rank) lib_seqs.append(lib_seq) s += "\n" + "\n".join(load_csv(None, lib_seqs, sep=" ")) + "\n" params = [("genome_size", genomesize), ("is_diploid", 0), ("mer_size", opts.K), ("num_prefix_blocks", 1), ("no_read_validation", 0), ("local_num_procs", opts.cpus)] s += "\n" + "\n".join(load_csv(None, params, sep=" ")) + "\n" cfgfile = "meraculous.config" write_file(cfgfile, s, tee=True) s = "~/export/meraculous/bin/run_meraculous.sh -c {0}"\ .format(cfgfile) runsh = "run.sh" write_file(runsh, s)
def deduplicate(args): """ %prog deduplicate fastafile Wraps `cd-hit-est` to remove duplicate sequences. """ p = OptionParser(deduplicate.__doc__) p.set_align(pctid=96, pctcov=0) p.add_option("--fast", default=False, action="store_true", help="Place sequence in the first cluster") p.add_option("--consensus", default=False, action="store_true", help="Compute consensus sequences") p.add_option("--reads", default=False, action="store_true", help="Use `cd-hit-454` to deduplicate [default: %default]") p.add_option("--samestrand", default=False, action="store_true", help="Enforce same strand alignment") p.set_home("cdhit") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args identity = opts.pctid / 100. fastafile, qualfile = fasta([fastafile, "--seqtk"]) ocmd = "cd-hit-454" if opts.reads else "cd-hit-est" cmd = op.join(opts.cdhit_home, ocmd) cmd += " -c {0}".format(identity) if ocmd == "cd-hit-est": cmd += " -d 0" # include complete defline if opts.samestrand: cmd += " -r 0" if not opts.fast: cmd += " -g 1" if opts.pctcov != 0: cmd += " -aL {0} -aS {0}".format(opts.pctcov / 100.) dd = fastafile + ".P{0}.cdhit".format(opts.pctid) clstr = dd + ".clstr" cmd += " -M 0 -T {0} -i {1} -o {2}".format(opts.cpus, fastafile, dd) if need_update(fastafile, (dd, clstr)): sh(cmd) if opts.consensus: cons = dd + ".consensus" cmd = op.join(opts.cdhit_home, "cdhit-cluster-consensus") cmd += " clustfile={0} fastafile={1} output={2} maxlen=1".\ format(clstr, fastafile, cons) if need_update((clstr, fastafile), cons): sh(cmd) return dd
def blasr(args): """ %prog blasr ref.fasta fofn Run blasr on a set of PacBio reads. This is based on a divide-and-conquer strategy described below. """ from jcvi.apps.grid import MakeManager from jcvi.utils.iter import grouper p = OptionParser(blasr.__doc__) p.set_cpus(cpus=8) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) reffasta, fofn = args flist = sorted([x.strip() for x in open(fofn)]) h5list = [] mm = MakeManager() for i, fl in enumerate(grouper(flist, 3)): chunkname = "chunk{0:03d}".format(i) fn = chunkname + ".fofn" h5 = chunkname + ".cmp.h5" fw = open(fn, "w") print >> fw, "\n".join(fl) fw.close() cmd = "pbalign {0} {1} {2}".format(fn, reffasta, h5) cmd += " --nproc {0} --forQuiver --tmpDir .".format(opts.cpus) mm.add((fn, reffasta), h5, cmd) h5list.append(h5) # Merge h5, sort and repack allh5 = "all.cmp.h5" tmph5 = "tmp.cmp.h5" cmd_merge = "cmph5tools.py merge --outFile {0}".format(allh5) cmd_merge += " " + " ".join(h5list) cmd_sort = "cmph5tools.py sort --deep {0} --tmpDir .".format(allh5) cmd_repack = "h5repack -f GZIP=1 {0} {1}".format(allh5, tmph5) cmd_repack += " && mv {0} {1}".format(tmph5, allh5) mm.add(h5list, allh5, [cmd_merge, cmd_sort, cmd_repack]) # Quiver pf = reffasta.rsplit(".", 1)[0] variantsgff = pf + ".variants.gff" consensusfasta = pf + ".consensus.fasta" cmd_faidx = "samtools faidx {0}".format(reffasta) cmd = "quiver -j 32 {0}".format(allh5) cmd += " -r {0} -o {1} -o {2}".format(reffasta, variantsgff, consensusfasta) mm.add(allh5, consensusfasta, [cmd_faidx, cmd]) mm.write()
def optimize(args): """ %prog optimize test.clm Optimize the contig order and orientation, based on CLM file. """ p = OptionParser(optimize.__doc__) p.add_option("--skiprecover", default=False, action="store_true", help="Do not import 'recover' contigs") p.add_option("--startover", default=False, action="store_true", help="Do not resume from existing tour file") p.add_option("--skipGA", default=False, action="store_true", help="Skip GA step") p.set_outfile(outfile=None) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) clmfile, = args startover = opts.startover runGA = not opts.skipGA cpus = opts.cpus # Load contact map clm = CLMFile(clmfile, skiprecover=opts.skiprecover) tourfile = opts.outfile or clmfile.rsplit(".", 1)[0] + ".tour" if startover: tourfile = None tour = clm.activate(tourfile=tourfile) fwtour = open(tourfile, "w") # Store INIT tour print_tour(fwtour, clm.tour, "INIT", clm.active_contigs, clm.oo, signs=clm.signs) if runGA: for phase in range(1, 3): tour = optimize_ordering(fwtour, clm, phase, cpus) tour = clm.prune_tour(tour, cpus) # Flip orientations phase = 1 while True: tag1, tag2 = optimize_orientations(fwtour, clm, phase, cpus) if tag1 == REJECT and tag2 == REJECT: logging.debug("Terminating ... no more {}".format(ACCEPT)) break phase += 1 fwtour.close()
def cluster(args): """ %prog cluster prefix fastqfiles Use `vsearch` to remove duplicate reads. This routine is heavily influenced by PyRAD: <https://github.com/dereneaton/pyrad>. """ p = OptionParser(cluster.__doc__) add_consensus_options(p) p.set_align(pctid=95) p.set_outdir() p.set_cpus() opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) prefix = args[0] fastqfiles = args[1:] cpus = opts.cpus pctid = opts.pctid mindepth = opts.mindepth minlength = opts.minlength fastafile, qualfile = fasta(fastqfiles + ["--seqtk", "--outdir={0}".format(opts.outdir), "--outfile={0}".format(prefix + ".fasta")]) prefix = op.join(opts.outdir, prefix) pf = prefix + ".P{0}".format(pctid) derepfile = prefix + ".derep" if need_update(fastafile, derepfile): derep(fastafile, derepfile, minlength, cpus) userfile = pf + ".u" notmatchedfile = pf + ".notmatched" if need_update(derepfile, userfile): cluster_smallmem(derepfile, userfile, notmatchedfile, minlength, pctid, cpus) clustfile = pf + ".clust" if need_update((derepfile, userfile, notmatchedfile), clustfile): makeclust(derepfile, userfile, notmatchedfile, clustfile, mindepth=mindepth) clustSfile = pf + ".clustS" if need_update(clustfile, clustSfile): parallel_musclewrap(clustfile, cpus) statsfile = pf + ".stats" if need_update(clustSfile, statsfile): makestats(clustSfile, statsfile, mindepth=mindepth)
def cufflinks(args): """ %prog cufflinks folder reference Run cufflinks on a folder containing tophat results. """ p = OptionParser(cufflinks.__doc__) p.add_option("--gtf", help="Reference annotation [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) folder, reference = args cpus = opts.cpus gtf = opts.gtf transcripts = "transcripts.gtf" mm = MakeManager() gtfs = [] for bam in iglob(folder, "*.bam"): pf = op.basename(bam).split(".")[0] outdir = pf + "_cufflinks" cmd = "cufflinks" cmd += " -o {0}".format(outdir) cmd += " -p {0}".format(cpus) if gtf: cmd += " -g {0}".format(gtf) cmd += " --frag-bias-correct {0}".format(reference) cmd += " --multi-read-correct" cmd += " {0}".format(bam) cgtf = op.join(outdir, transcripts) mm.add(bam, cgtf, cmd) gtfs.append(cgtf) assemblylist = "assembly_list.txt" cmd = 'find . -name "{0}" > {1}'.format(transcripts, assemblylist) mm.add(gtfs, assemblylist, cmd) mergedgtf = "merged/merged.gtf" cmd = "cuffmerge" cmd += " -o merged" cmd += " -p {0}".format(cpus) if gtf: cmd += " -g {0}".format(gtf) cmd += " -s {0}".format(reference) cmd += " {0}".format(assemblylist) mm.add(assemblylist, mergedgtf, cmd) mm.write()
def mcluster(args): """ %prog mcluster *.consensus Cluster across samples using consensus sequences. """ p = OptionParser(mcluster.__doc__) add_consensus_options(p) p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) consensusfiles = args minlength = opts.minlength cpus = opts.cpus pf = opts.prefix pctid = find_pctid(consensusfiles) pf += ".P{0}".format(pctid) consensusfile = pf + ".consensus.fasta" if need_update(consensusfiles, consensusfile): fw_cons = must_open(consensusfile, "w") totalseqs = 0 for cf in consensusfiles: nseqs = 0 s = op.basename(cf).split(".")[0] for name, seq in parse_fasta(cf): name = ".".join((s, name)) print(">{0}\n{1}".format(name, seq), file=fw_cons) nseqs += 1 logging.debug("Read `{0}`: {1} seqs".format(cf, nseqs)) totalseqs += nseqs logging.debug("Total: {0} seqs".format(totalseqs)) fw_cons.close() userfile = pf + ".u" notmatchedfile = pf + ".notmatched" if need_update(consensusfile, userfile): cluster_smallmem(consensusfile, userfile, notmatchedfile, minlength, pctid, cpus) clustfile = pf + ".clust" if need_update((consensusfile, userfile, notmatchedfile), clustfile): makeclust(consensusfile, userfile, notmatchedfile, clustfile) clustSfile = pf + ".clustS" if need_update(clustfile, clustSfile): parallel_musclewrap(clustfile, cpus, minsamp=opts.minsamp)
def compilevcf(args): """ %prog compilevcf samples.csv Compile vcf results into master spreadsheet. """ p = OptionParser(compilevcf.__doc__) p.add_option("--db", default="hg38", help="Use these lobSTR db") p.add_option("--stutter", default=False, action="store_true", help="Count stutter reads on chrY") p.add_option("--nofilter", default=False, action="store_true", help="Do not filter the variants") p.set_home("lobstr") p.set_cpus() p.set_aws_opts(store="hli-mv-data-science/htang/str-data") opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) samples, = args workdir = opts.workdir store = opts.output_path stutter = opts.stutter cleanup = not opts.nocleanup filtered = not opts.nofilter dbs = opts.db.split(",") cwd = os.getcwd() mkdir(workdir) os.chdir(workdir) samples = op.join(cwd, samples) stridsfile = "STR.ids" vcffiles = [x.strip() for x in must_open(samples)] if not op.exists(stridsfile): ids = [] for db in dbs: ids.extend(STRFile(opts.lobstr_home, db=db).ids) uids = uniqify(ids) logging.debug("Combined: {} Unique: {}".format(len(ids), len(uids))) fw = open(stridsfile, "w") print >> fw, "\n".join(uids) fw.close() run_args = [(x, filtered, cleanup, store, stutter) for x in vcffiles] cpus = min(opts.cpus, len(run_args)) p = Pool(processes=cpus) for res in p.map_async(run_compile, run_args).get(): continue
def align(args): """ %prog align clustfile Align clustfile to clustSfile. Useful for benchmarking aligners. """ p = OptionParser(align.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) clustfile, = args parallel_musclewrap(clustfile, opts.cpus)
def mappability(args): """ %prog mappability reference.fasta Generate 50mer mappability for reference genome. Commands are based on gem mapper. See instructions: <https://github.com/xuefzhao/Reference.Mappability> """ p = OptionParser(mappability.__doc__) p.add_option("--mer", default=50, type="int", help="User mer size") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) ref, = args K = opts.mer pf = ref.rsplit(".", 1)[0] mm = MakeManager() gem = pf + ".gem" cmd = "gem-indexer -i {} -o {}".format(ref, pf) mm.add(ref, gem, cmd) mer = pf + ".{}mer".format(K) mapb = mer + ".mappability" cmd = "gem-mappability -I {} -l {} -o {} -T {}".\ format(gem, K, mer, opts.cpus) mm.add(gem, mapb, cmd) wig = mer + ".wig" cmd = "gem-2-wig -I {} -i {} -o {}".format(gem, mapb, mer) mm.add(mapb, wig, cmd) bw = mer + ".bw" cmd = "wigToBigWig {} {}.sizes {}".format(wig, mer, bw) mm.add(wig, bw, cmd) bg = mer + ".bedGraph" cmd = "bigWigToBedGraph {} {}".format(bw, bg) mm.add(bw, bg, cmd) merged = mer + ".filtered-1.merge.bed" cmd = "python -m jcvi.formats.bed filterbedgraph {} 1".format(bg) mm.add(bg, merged, cmd) mm.write()
def layout(args): """ %prog layout query.subject.simple query.seqids subject.seqids Compute optimal seqids order in a second genome, based on seqids on one genome, given the pairwise blocks in .simple format. """ from jcvi.algorithms.ec import GA_setup, GA_run p = OptionParser(layout.__doc__) p.set_beds() p.set_cpus(cpus=32) opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) simplefile, qseqids, sseqids = args qbed, sbed, qorder, sorder, is_self = check_beds(simplefile, p, opts) qseqids = qseqids.strip().split(",") sseqids = sseqids.strip().split(",") qseqids_ii = dict((s, i) for i, s in enumerate(qseqids)) sseqids_ii = dict((s, i) for i, s in enumerate(sseqids)) blocks = SimpleFile(simplefile).blocks scores = defaultdict(int) for a, b, c, d, score, orientation, hl in blocks: qi, q = qorder[a] si, s = sorder[c] qseqid, sseqid = q.seqid, s.seqid if sseqid not in sseqids: continue scores[sseqids_ii[sseqid], qseqid] += score data = [] for (a, b), score in sorted(scores.items()): if b not in qseqids_ii: continue data.append((qseqids_ii[b], score)) tour = range(len(qseqids)) toolbox = GA_setup(tour) toolbox.register("evaluate", colinear_evaluate_weights, data=data) tour, fitness = GA_run(toolbox, ngen=100, npop=100, cpus=opts.cpus) tour = [qseqids[x] for x in tour] print ",".join(tour)
def bes(args): """ %prog bes bacfasta clonename Use the clone name to download BES gss sequences from Genbank, map and then visualize. """ from jcvi.apps.align import run_blat p = OptionParser(bes.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bacfasta, clonename = args entrez([clonename, "--database=nucgss", "--skipcheck"]) besfasta = clonename + ".fasta" blatfile = clonename + ".bes.blat" run_blat(infile=besfasta, outfile=blatfile, db=bacfasta, \ pctid=95, hitlen=100, cpus=opts.cpus) aid, asize = Fasta(bacfasta).itersizes().next() width = 50 msg = "=" * width msg += " " + aid print >> sys.stderr, msg ratio = width * 1. / asize _ = lambda x: int(round(x * ratio, 0)) blasts = [BlastLine(x) for x in open(blatfile)] for b in blasts: if b.orientation == '+': msg = " " * _(b.sstart) + "->" else: msg = " " * (_(b.sstop) - 2) + "<-" msg += " " * (width - len(msg) + 2) msg += b.query if b.orientation == '+': msg += " (hang={0})".format(b.sstart - 1) else: msg += " (hang={0})".format(asize - b.sstop) print >> sys.stderr, msg
def alignextend(args): """ %prog alignextend ref.fasta read.1.fastq read.2.fastq Wrapper around AMOS alignextend. """ choices = "prepare,align,filter,rmdup,genreads".split(",") p = OptionParser(alignextend.__doc__) p.add_option("--nosuffix", default=False, action="store_true", help="Do not add /1/2 suffix to the read [default: %default]") p.add_option("--rc", default=False, action="store_true", help="Reverse complement the reads before alignment") p.add_option("--len", default=100, type="int", help="Extend to this length") p.add_option("--stage", default="prepare", choices=choices, help="Start from certain stage") p.add_option("--dup", default=10, type="int", help="Filter duplicates with coordinates within this distance") p.add_option("--maxdiff", default=1, type="int", help="Maximum number of differences") p.set_home("amos") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) ref, r1, r2 = args pf = op.basename(r1).split(".")[0] cmd = op.join(opts.amos_home, "src/Experimental/alignextend.pl") if not opts.nosuffix: cmd += " -suffix" bwa_idx = "{0}.ref.fa.sa".format(pf) if not need_update(ref, bwa_idx): cmd += " -noindex" cmd += " -threads {0}".format(opts.cpus) offset = guessoffset([r1]) if offset == 64: cmd += " -I" if opts.rc: cmd += " -rc" cmd += " -allow -len {0} -dup {1}".format(opts.len, opts.dup) cmd += " -min {0} -max {1}".format(2 * opts.len, 20 * opts.len) cmd += " -maxdiff {0}".format(opts.maxdiff) cmd += " -stage {0}".format(opts.stage) cmd += " ".join(("", pf, ref, r1, r2)) sh(cmd)
def gmap(args): """ %prog gmap database.fasta fastafile Wrapper for `gmap`. """ p = OptionParser(gmap.__doc__) p.add_option("--cross", default=False, action="store_true", help="Cross-species alignment") p.add_option( "--npaths", default=0, type="int", help="Maximum number of paths to show." " If set to 0, prints two paths if chimera" " detected, else one.", ) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) dbfile, fastafile = args assert op.exists(dbfile) and op.exists(fastafile) prefix = get_prefix(fastafile, dbfile) logfile = prefix + ".log" gmapfile = prefix + ".gmap.gff3" if not need_update((dbfile, fastafile), gmapfile): logging.error("`{0}` exists. `gmap` already run.".format(gmapfile)) else: dbdir, dbname = check_index(dbfile) cmd = "gmap -D {0} -d {1}".format(dbdir, dbname) cmd += " -f 2 --intronlength=100000" # Output format 2 cmd += " -t {0}".format(opts.cpus) cmd += " --npaths {0}".format(opts.npaths) if opts.cross: cmd += " --cross-species" cmd += " " + fastafile sh(cmd, outfile=gmapfile, errfile=logfile) return gmapfile, logfile
def clean(args): """ %prog clean 1.fastq 2.fastq [insertsize] Clean and dedup paired FASTQ files. """ p = OptionParser(clean.__doc__) p.add_option("-a", default=0, type="int", help="Trim length at 5' end [default: %default]") p.add_option("-b", default=50, type="int", help="Trim length at 3' end [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) == 2: p1, p2 = args size = get_size(p1) elif len(args) == 3: p1, p2, size = args size = int(size) else: sys.exit(not p.print_help()) pf = p1.split(".")[0] cpus = opts.cpus offset = guessoffset([p1]) a, b = opts.a, opts.b p1_clean = p1 + ".clean" p1_cleangz = p1_clean + ".gz" p2_clean = p2 + ".clean" p2_cleangz = p2_clean + ".gz" if need_update([p1, p2], [p1_cleangz, p2_cleangz]): cmd = "SOAPfilter_v2.0 -t {0} -m 2000000 -p -y -z -g".format(cpus) cmd += " -q {0} -w 10 -B 50 -f 0".format(offset) cmd += " -l {0} -a {1} -b {2} -c {1} -d {2}".format(size, a, b, a, b) cmd += " {0} {1} {2}.clean.stat {3} {4}".\ format(p1, p2, pf, p1_clean, p2_clean) sh(cmd)
def mergebam(args): """ %prog mergebam dir1 dir2 homo_outdir or %prog mergebam dir1 dir2/20.bam het_outdir Merge sets of BAMs to make diploid. Two modes: - Homozygous mode: pair-up the bams in the two folders and merge - Heterozygous mode: pair the bams in first folder with a particular bam """ p = OptionParser(mergebam.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) idir1, idir2, outdir = args dir1 = [idir1] if idir1.endswith(".bam") else iglob(idir1, "*.bam") dir2 = [idir2] if idir2.endswith(".bam") else iglob(idir2, "*.bam") nbams1 = len(dir1) nbams2 = len(dir2) # Make sure more or the same number of bams in first pile if nbams1 < nbams2: dir1, dir2 = dir2, dir1 if nbams1 == nbams2: logging.debug("Homozygous mode") elif nbams1 > nbams2: assert nbams2 == 1, "Second pile must contain a single bam" dir2 = [idir2] * nbams1 assert len(dir1) == len(dir2), "Two piles must contain same number of bams" cmd = "samtools merge {} {} {} && samtools index {}" cmds = [] mkdir(outdir) for a, b in zip(dir1, dir2): ia = op.basename(a).split(".")[0] ib = op.basename(b).split(".")[0] outfile = op.join(outdir, "{}_{}.bam".format(ia, ib)) cmds.append(cmd.format(outfile, a, b, outfile)) p = Parallel(cmds, cpus=opts.cpus) p.run()
def fill(args): """ %prog fill frag_reads_corr.fastb Run FillFragments on `frag_reads_corr.fastb`. """ p = OptionParser(fill.__doc__) p.add_option( "--stretch", default=3, type="int", help="MAX_STRETCH to pass to FillFragments", ) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) (fastb, ) = args assert fastb == "frag_reads_corr.fastb" pcfile = "frag_reads_corr.k28.pc.info" nthreads = " NUM_THREADS={0}".format(opts.cpus) maxstretch = " MAX_STRETCH={0}".format(opts.stretch) if need_update(fastb, pcfile): cmd = "PathReads READS_IN=frag_reads_corr" cmd += nthreads sh(cmd) filledfastb = "filled_reads.fastb" if need_update(pcfile, filledfastb): cmd = "FillFragments PAIRS_OUT=frag_reads_corr_cpd" cmd += " PRECORRECT_LIBSTATS=True" cmd += maxstretch cmd += nthreads sh(cmd) filledfasta = "filled_reads.fasta" if need_update(filledfastb, filledfasta): cmd = "Fastb2Fasta IN=filled_reads.fastb OUT=filled_reads.fasta" sh(cmd)
def cp(args): """ %prog cp "s3://hli-mv-data-science/htang/str/*.csv" . Copy files to folder. Accepts list of s3 addresses as input. """ p = OptionParser(cp.__doc__) p.add_option("--force", default=False, action="store_true", help="Force overwrite if exists") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) store, folder = args force = opts.force cpus = opts.cpus if op.exists(store): contents = [x.strip().split(",") for x in open(store)] else: contents = glob_s3(store) tasks = [] for c in contents: if isinstance(c, basestring): oc = op.basename(c) tc = op.join(folder, oc) else: if len(c) == 2: c, tc = c else: c, = c tc = op.basename(c) tasks.append((c, tc, force)) worker_pool = Pool(cpus) worker_pool.map(worker, tasks) worker_pool.close() worker_pool.join()
def nucmer(args): """ %prog nucmer ref.fasta query.fasta Run NUCMER using query against reference. Parallel implementation derived from: <https://github.com/fritzsedlazeck/sge_mummer> """ from itertools import product from jcvi.apps.grid import MakeManager from jcvi.formats.base import split p = OptionParser(nucmer.__doc__) p.add_option("--chunks", type="int", help="Split both query and subject into chunks") p.set_params(prog="nucmer", params="-g 5000 -l 24 -c 500") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) ref, query = args cpus = opts.cpus nrefs = nqueries = opts.chunks or int(cpus**.5) refdir = ref.split(".")[0] + "-outdir" querydir = query.split(".")[0] + "-outdir" reflist = split([ref, refdir, str(nrefs)]).names querylist = split([query, querydir, str(nqueries)]).names mm = MakeManager() for i, (r, q) in enumerate(product(reflist, querylist)): pf = "{0:04d}".format(i) cmd = "nucmer -maxmatch" cmd += " {0}".format(opts.extra) cmd += " {0} {1} -p {2}".format(r, q, pf) deltafile = pf + ".delta" mm.add((r, q), deltafile, cmd) print cmd mm.write()
def blast(args): """ %prog blast ref.fasta query.fasta Calls blast and then filter the BLAST hits. Default is megablast. """ task_choices = ("blastn", "blastn-short", "dc-megablast", \ "megablast", "vecscreen") p = OptionParser(blast.__doc__) p.set_align(pctid=0, evalue=.01) p.add_option("--wordsize", type="int", help="Word size [default: %default]") p.add_option("--best", default=1, type="int", help="Only look for best N hits [default: %default]") p.add_option("--task", default="megablast", choices=task_choices, help="Task of the blastn [default: %default]") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) reffasta, queryfasta = args blastfile = get_outfile(reffasta, queryfasta) run_megablast(infile=queryfasta, outfile=blastfile, db=reffasta, wordsize=opts.wordsize, pctid=opts.pctid, evalue=opts.evalue, hitlen=None, best=opts.best, task=opts.task, cpus=opts.cpus) return blastfile
def compile(args): """ %prog compile samples.csv Compile vcf results into master spreadsheet. """ p = OptionParser(compile.__doc__) p.add_option("--db", default="hg38", help="Use these lobSTR db") p.set_home("lobstr") p.set_cpus() p.set_aws_opts(store="hli-mv-data-science/htang/str") opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) samples, = args workdir = opts.workdir dbs = opts.db.split(",") cwd = os.getcwd() mkdir(workdir) os.chdir(workdir) samples = op.join(cwd, samples) stridsfile = "STR.ids" vcffiles = [x.strip() for x in must_open(samples)] if not op.exists(stridsfile): ids = [] for db in dbs: ids.extend(STRFile(opts.lobstr_home, db=db).ids) uids = uniqify(ids) logging.debug("Combined: {} Unique: {}".format(len(ids), len(uids))) fw = open(stridsfile, "w") print >> fw, "\n".join(uids) fw.close() p = Pool(processes=opts.cpus) run_args = [(x, opts.store, opts.cleanup) for x in vcffiles] for res in p.map_async(run, run_args).get(): continue
def close(args): """ %prog close scaffolds.fasta PE*.fastq Run GapFiller to fill gaps. """ p = OptionParser(close.__doc__) p.set_home("gapfiller") p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) scaffolds = args[0] libtxt = write_libraries(args[1:], aligner="bwa") cmd = "perl " + op.join(opts.gapfiller_home, "GapFiller.pl") cmd += " -l {0} -s {1} -T {2}".format(libtxt, scaffolds, opts.cpus) runsh = "run.sh" write_file(runsh, cmd)
def scaffold(args): """ %prog scaffold contigs.fasta MP*.fastq Run SSPACE scaffolding. """ p = OptionParser(scaffold.__doc__) p.set_home("sspace") p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) contigs = args[0] libtxt = write_libraries(args[1:]) cmd = "perl " + op.join(opts.sspace_home, "SSPACE_Basic_v2.0.pl") cmd += " -l {0} -s {1} -T {2}".format(libtxt, contigs, opts.cpus) runsh = "run.sh" write_file(runsh, cmd)
def blat(args): """ %prog blat old.fasta new.fasta Generate psl file using blat. """ p = OptionParser(blat.__doc__) p.add_option( "--minscore", default=100, type="int", help="Matches minus mismatches gap penalty", ) p.add_option( "--minid", default=98, type="int", help="Minimum sequence identity", ) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) oldfasta, newfasta = args twobitfiles = [] for fastafile in args: tbfile = faToTwoBit(fastafile) twobitfiles.append(tbfile) oldtwobit, newtwobit = twobitfiles cmd = "pblat -threads={0}".format(opts.cpus) if which("pblat") else "blat" cmd += " {0} {1}".format(oldtwobit, newfasta) cmd += " -tileSize=12 -minScore={0} -minIdentity={1} ".format( opts.minscore, opts.minid) pslfile = "{0}.{1}.psl".format(*(op.basename(x).split(".")[0] for x in (newfasta, oldfasta))) cmd += pslfile sh(cmd)
def prepare(args): """ %prog prepare alignAssembly.config est.fasta ref.fasta Generate PASA run script. """ p = OptionParser(prepare.__doc__) p.set_home("pasa") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 3: sys.exit(not p.print_help()) cfg, est, ref = args phome = opts.pasa_home cmd = op.join(phome, "scripts/Launch_PASA_pipeline.pl") cmd += " -c {0} --CPU {1}".format(cfg, opts.cpus) cmd += " -C -R --ALIGNERS blat,gmap" cmd += " -t {0} -g {1}".format(est, ref) runfile = "run.sh" write_file(runfile, cmd, meta="run script")
def correct(args): """ %prog correct *.fastq Correct reads using ErrorCorrection. Only PE will be used to build the K-mer table. """ p = OptionParser(correct.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) lstfile = "reads2cor.lst" fw = open(lstfile, "w") print("\n".join(x for x in args if x[:2] == "PE"), file=fw) fw.close() p1 = args[0] offset = guessoffset([p1]) cpus = opts.cpus freq = "output.freq.cz" freqlen = freq + ".len" if need_update(args, (freq, freqlen)): cmd = "KmerFreq_AR_v2.0 -k 17 -c -1 -q {0}".format(offset) cmd += " -m 1 -t {0}".format(cpus) cmd += " -p output {0}".format(lstfile) sh(cmd) fw = open(lstfile, "w") print("\n".join(args), file=fw) fw.close() cmd = "Corrector_AR_v2.0 -k 17 -l 3 -m 5 -c 5 -a 0 -e 1 -w 0 -r 45" cmd += " -Q {0} -q 30 -x 8 -t {1} -o 1 ".format(offset, cpus) cmd += " {0} {1} {2}".format(freq, freqlen, lstfile) sh(cmd)
def blat(args): """ %prog blat ref.fasta query.fasta Calls blat and filters BLAST hits. """ p = OptionParser(blat.__doc__) p.set_align(pctid=95, hitlen=30) p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) reffasta, queryfasta = args blastfile = get_outfile(reffasta, queryfasta, suffix="blat") run_blat(infile=queryfasta, outfile=blastfile, db=reffasta, pctid=opts.pctid, hitlen=opts.hitlen, cpus=opts.cpus, overwrite=False) return blastfile
def kmc(args): """ %prog kmc folder Run kmc3 on Illumina reads. """ p = OptionParser(kmc.__doc__) p.add_option("-k", default=21, type="int", help="Kmer size") p.add_option("--ci", default=2, type="int", help="Minimum value of a counter") p.add_option("--cs", default=2, type="int", help="Maximal value of a counter") p.add_option("--single", default=False, action="store_true", help="Input is single-end data, only one FASTQ") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) folder, = args K = opts.k n = 1 if opts.single else 2 mm = MakeManager() for p, pf in iter_project(folder, n=n, commonprefix=False): pf = pf.split("_")[0] + ".ms{}".format(K) infiles = pf + ".infiles" fw = open(infiles, "w") print >> fw, "\n".join(p) fw.close() cmd = "kmc -k{} -m64 -t{}".format(K, opts.cpus) cmd += " -ci{} -cs{}".format(opts.ci, opts.cs) cmd += " @{} {} .".format(infiles, pf) outfile = pf + ".kmc_suf" mm.add(p, outfile, cmd) mm.write()
def genemark(args): """ %prog genemark species fastafile Train GENEMARK model given fastafile. GENEMARK self-trains so no trainig model gff file is needed. """ p = OptionParser(genemark.__doc__) p.add_option("--junctions", help="Path to `junctions.bed` from Tophat2") p.set_home("gmes") p.set_cpus(cpus=32) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) species, fastafile = args junctions = opts.junctions mhome = opts.gmes_home license = op.expanduser("~/.gm_key") assert op.exists(license), "License key ({0}) not found!".format(license) cmd = "{0}/gmes_petap.pl --sequence {1}".format(mhome, fastafile) cmd += " --cores {0}".format(opts.cpus) if junctions: intronsgff = "introns.gff" if need_update(junctions, intronsgff): jcmd = "{0}/bet_to_gff.pl".format(mhome) jcmd += " --bed {0} --gff {1} --label Tophat2".\ format(junctions, intronsgff) sh(jcmd) cmd += " --ET {0} --et_score 10".format(intronsgff) else: cmd += " --ES" sh(cmd) logging.debug("GENEMARK matrix written to `output/gmhmm.mod")
def deduplicate(args): """ %prog deduplicate fastafile Wraps `cd-hit-est` to remove duplicate sequences. """ p = OptionParser(deduplicate.__doc__) p.set_align(pctid=98) p.add_option("--reads", default=False, action="store_true", help="Use `cd-hit-454` to deduplicate [default: %default]") p.add_option("--samestrand", default=False, action="store_true", help="Enforce same strand alignment") p.set_home("cdhit") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) fastafile, = args identity = opts.pctid / 100. cmd = "cd-hit-454" if opts.reads else "cd-hit-est" cmd = op.join(opts.cdhit_home, cmd) cmd += " -c {0}".format(identity) cmd += " -d 0" # include complete defline if opts.samestrand: cmd += " -r 0" cmd += " -M 0 -T {0} -i {1} -o {1}.cdhit".format(opts.cpus, fastafile) sh(cmd) dd = fastafile + ".cdhit" return dd
def meryl(args): """ %prog meryl folder Run meryl on Illumina reads. """ p = OptionParser(meryl.__doc__) p.add_option("-k", default=19, type="int", help="Kmer size") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) (folder, ) = args K = opts.k cpus = opts.cpus mm = MakeManager() for p, pf in iter_project(folder): cmds = [] mss = [] for i, ip in enumerate(p): ms = "{}{}.ms{}".format(pf, i + 1, K) mss.append(ms) cmd = "meryl -B -C -m {} -threads {}".format(K, cpus) cmd += " -s {} -o {}".format(ip, ms) cmds.append(cmd) ams, bms = mss pms = "{}.ms{}".format(pf, K) cmd = "meryl -M add -s {} -s {} -o {}".format(ams, bms, pms) cmds.append(cmd) cmd = "rm -f {}.mcdat {}.mcidx {}.mcdat {}.mcidx".format( ams, ams, bms, bms) cmds.append(cmd) mm.add(p, pms + ".mcdat", cmds) mm.write()
def compare(args): """ %prog compare NA12878_array_hg38.bed *.seg Compare cnv output to known ground truths. """ p = OptionParser(compare.__doc__) p.set_cpus() opts, args = p.parse_args(args) if len(args) < 2: sys.exit(not p.print_help()) truths = args[0] cnvoutputs = args[1:] cpus = min(len(cnvoutputs), opts.cpus) p = Pool(processes=cpus) results = [] files = [(x, truths) for x in cnvoutputs] r = p.map_async(compare_worker, files, callback=results.append) r.wait() for res in results: print("\n".join(res))
def scaffold(args): """ %prog scaffold contigs.fasta MP*.fastq Run SSPACE scaffolding. """ p = OptionParser(scaffold.__doc__) p.set_aligner(aligner="bwa") p.set_home("sspace") p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) contigs = args[0] libtxt = write_libraries(args[1:], aligner=opts.aligner) # Requires getopts.pl which may be missing download("http://web.vims.edu/bridge/bridge2/aw/lib/getopts.pl") cmd = "perl " + op.join(opts.sspace_home, "SSPACE_Standard_v3.0.pl") cmd += " -l {0} -s {1} -T {2}".format(libtxt, contigs, opts.cpus) runsh = "run.sh" write_file(runsh, cmd)
def batchlobstr(args): """ %prog batchlobstr bamlist Run lobSTR on a list of BAMs. The corresponding batch command for TREDPARSE: $ tred.py --toy bamlist --haploid CHR4 --workdir tredparse_results """ p = OptionParser(batchlobstr.__doc__) p.add_option("--haploid", default="chrY,chrM", help="Use haploid model for these chromosomes") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) bamlist, = args cmd = "python -m jcvi.variation.str lobstr TOY" cmd += " --input_bam_path {}" cmd += " --haploid {}".format(opts.haploid) cmds = [cmd.format(x.strip()) for x in open(bamlist).readlines()] p = Parallel(cmds, cpus=opts.cpus) p.run()
def density(args): """ %prog density test.clm Estimate link density of contigs. """ p = OptionParser(density.__doc__) p.add_option("--save", default=False, action="store_true", help="Write log densitites of contigs to file") p.set_cpus() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) clmfile, = args clm = CLMFile(clmfile) pf = clmfile.rsplit(".", 1)[0] if opts.save: logdensities = clm.calculate_densities() densityfile = pf + ".density" fw = open(densityfile, "w") for name, logd in logdensities.items(): s = clm.tig_to_size[name] print >> fw, "\t".join(str(x) for x in (name, s, logd)) fw.close() logging.debug("Density written to `{}`".format(densityfile)) tourfile = clmfile.rsplit(".", 1)[0] + ".tour" tour = clm.activate(tourfile=tourfile, backuptour=False) clm.flip_all(tour) clm.flip_whole(tour) clm.flip_one(tour)
def jellyfish(args): """ %prog jellyfish [*.fastq|*.fasta] Run jellyfish to dump histogram to be used in kmer.histogram(). """ from jcvi.apps.base import getfilesize from jcvi.utils.cbook import human_size p = OptionParser(jellyfish.__doc__) p.add_option("-K", default=23, type="int", help="K-mer size") p.add_option( "--coverage", default=40, type="int", help="Expected sequence coverage", ) p.add_option("--prefix", default="jf", help="Database prefix") p.add_option( "--nohist", default=False, action="store_true", help="Do not print histogram", ) p.set_home("jellyfish") p.set_cpus() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) fastqfiles = args K = opts.K coverage = opts.coverage totalfilesize = sum(getfilesize(x) for x in fastqfiles) fq = fastqfiles[0] pf = opts.prefix gzip = fq.endswith(".gz") hashsize = totalfilesize / coverage logging.debug("Total file size: {0}, hashsize (-s): {1}".format( human_size(totalfilesize, a_kilobyte_is_1024_bytes=True), hashsize)) jfpf = "{0}-K{1}".format(pf, K) jfdb = jfpf fastqfiles = " ".join(fastqfiles) jfcmd = op.join(opts.jellyfish_home, "jellyfish") cmd = jfcmd cmd += " count -t {0} -C -o {1}".format(opts.cpus, jfpf) cmd += " -s {0} -m {1}".format(hashsize, K) if gzip: cmd = "gzip -dc {0} | ".format(fastqfiles) + cmd + " /dev/fd/0" else: cmd += " " + fastqfiles if need_update(fastqfiles, jfdb): sh(cmd) if opts.nohist: return jfhisto = jfpf + ".histogram" cmd = jfcmd + " histo -t 64 {0} -o {1}".format(jfdb, jfhisto) if need_update(jfdb, jfhisto): sh(cmd)