def convert(args): """ %prog convert in.fastq out.fastq illumina fastq quality encoding uses offset 64, and sanger uses 33. This script creates a new file with the correct encoding """ p = OptionParser(convert.__doc__) p.set_phred() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) infastq, outfastq = args phred = opts.phred or str(guessoffset([infastq])) fin = "illumina" if phred == "64" else "sanger" fout = "sanger" if phred == "64" else "illumina" seqret = "seqret" if infastq.endswith(".gz"): cmd = "zcat {0} | ".format(infastq) cmd += seqret + " fastq-{0}::stdin fastq-{1}::stdout".format(fin, fout) else: cmd = seqret + " fastq-{0}::{1} fastq-{2}::stdout".format(fin, infastq, fout) sh(cmd, outfile=outfastq) return outfastq
def fastq(args): """ %prog fastq fastqfile Convert reads formatted as FASTQ file, and convert to CA frg file. """ from jcvi.formats.fastq import guessoffset p = OptionParser(fastq.__doc__) p.add_option( "--outtie", dest="outtie", default=False, action="store_true", help="Are these outie reads?", ) p.set_phred() p.set_size() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(p.print_help()) fastqfiles = [get_abs_path(x) for x in args] size = opts.size outtie = opts.outtie if size > 1000 and (not outtie): logging.debug( "[warn] long insert size {0} but not outtie".format(size)) mated = size != 0 libname = op.basename(args[0]).split(".")[0] libname = libname.replace("_1_sequence", "") frgfile = libname + ".frg" mean, sv = get_mean_sv(opts.size) cmd = "fastqToCA" cmd += " -libraryname {0} ".format(libname) fastqs = " ".join("-reads {0}".format(x) for x in fastqfiles) if mated: assert len(args) in ( 1, 2), "you need one or two fastq files for mated library" fastqs = "-mates {0}".format(",".join(fastqfiles)) cmd += "-insertsize {0} {1} ".format(mean, sv) cmd += fastqs offset = int(opts.phred) if opts.phred else guessoffset([fastqfiles[0]]) illumina = offset == 64 if illumina: cmd += " -type illumina" if outtie: cmd += " -outtie" sh(cmd, outfile=frgfile)
def tophat(args): """ %prog tophat folder reference Run tophat on a folder of reads. """ from jcvi.apps.bowtie import check_index from jcvi.formats.fastq import guessoffset p = OptionParser(tophat.__doc__) p.add_option("--gtf", help="Reference annotation [default: %default]") p.add_option("--single", default=False, action="store_true", help="Single end mapping") p.add_option("--intron", default=15000, type="int", help="Max intron size [default: %default]") p.add_option("--dist", default=-50, type="int", help="Mate inner distance [default: %default]") p.add_option("--stdev", default=50, type="int", help="Mate standard deviation [default: %default]") p.set_phred() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) num = 1 if opts.single else 2 folder, reference = args reference = check_index(reference) for p, prefix in iter_project(folder, n=num): outdir = "{0}_tophat".format(prefix) outfile = op.join(outdir, "accepted_hits.bam") if op.exists(outfile): logging.debug("File `{0}` found. Skipping.".format(outfile)) continue cmd = "tophat -p {0}".format(opts.cpus) if opts.gtf: cmd += " -G {0}".format(opts.gtf) cmd += " -o {0}".format(outdir) if num == 1: # Single-end a, = p else: # Paired-end a, b = p cmd += " --max-intron-length {0}".format(opts.intron) cmd += " --mate-inner-dist {0}".format(opts.dist) cmd += " --mate-std-dev {0}".format(opts.stdev) phred = opts.phred or str(guessoffset([a])) if phred == "64": cmd += " --phred64-quals" cmd += " {0} {1}".format(reference, " ".join(p)) sh(cmd)
def fastq(args): """ %prog fastq fastqfile Convert reads formatted as FASTQ file, and convert to CA frg file. """ from jcvi.formats.fastq import guessoffset p = OptionParser(fastq.__doc__) p.add_option( "--outtie", dest="outtie", default=False, action="store_true", help="Are these outie reads? [default: %default]" ) p.set_phred() p.set_size() opts, args = p.parse_args(args) if len(args) < 1: sys.exit(p.print_help()) fastqfiles = [get_abs_path(x) for x in args] size = opts.size outtie = opts.outtie if size > 1000 and (not outtie): logging.debug("[warn] long insert size {0} but not outtie".format(size)) mated = size != 0 libname = op.basename(args[0]).split(".")[0] libname = libname.replace("_1_sequence", "") frgfile = libname + ".frg" mean, sv = get_mean_sv(opts.size) cmd = "fastqToCA" cmd += " -libraryname {0} ".format(libname) fastqs = " ".join("-reads {0}".format(x) for x in fastqfiles) if mated: assert len(args) in (1, 2), "you need one or two fastq files for mated library" fastqs = "-mates {0}".format(",".join(fastqfiles)) cmd += "-insertsize {0} {1} ".format(mean, sv) cmd += fastqs offset = int(opts.phred) if opts.phred else guessoffset([fastqfiles[0]]) illumina = offset == 64 if illumina: cmd += " -type illumina" if outtie: cmd += " -outtie" sh(cmd, outfile=frgfile)
def convert(args): """ %prog convert in.fastq illumina fastq quality encoding uses offset 64, and sanger uses 33. This script creates a new file with the correct encoding. Output gzipped file if input is also gzipped. """ p = OptionParser(convert.__doc__) p.set_phred() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) infastq, = args phred = opts.phred or str(guessoffset([infastq])) ophred = {"64": "33", "33": "64"}[phred] gz = infastq.endswith(".gz") outfastq = infastq.rsplit(".", 1)[0] if gz else infastq pf, sf = outfastq.rsplit(".", 1) outfastq = "{0}.q{1}.{2}".format(pf, ophred, sf) if gz: outfastq += ".gz" fin = "illumina" if phred == "64" else "sanger" fout = "sanger" if phred == "64" else "illumina" seqret = "seqret" if infastq.endswith(".gz"): cmd = "zcat {0} | ".format(infastq) cmd += seqret + " fastq-{0}::stdin fastq-{1}::stdout".\ format(fin, fout) else: cmd = seqret + " fastq-{0}::{1} fastq-{2}::stdout".\ format(fin, infastq, fout) sh(cmd, outfile=outfastq) return outfastq
def convert(args): """ %prog convert in.fastq illumina fastq quality encoding uses offset 64, and sanger uses 33. This script creates a new file with the correct encoding. Output gzipped file if input is also gzipped. """ p = OptionParser(convert.__doc__) p.set_phred() opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) infastq, = args phred = opts.phred or str(guessoffset([infastq])) ophred = {"64": "33", "33": "64"}[phred] gz = infastq.endswith(".gz") outfastq = infastq.rsplit(".", 1)[0] if gz else infastq pf, sf = outfastq.rsplit(".", 1) outfastq = "{0}.q{1}.{2}".format(pf, ophred, sf) if gz: outfastq += ".gz" fin = "illumina" if phred == "64" else "sanger" fout = "sanger" if phred == "64" else "illumina" seqret = "seqret" if infastq.endswith(".gz"): cmd = "zcat {0} | ".format(infastq) cmd += seqret + " fastq-{0}::stdin fastq-{1}::stdout".\ format(fin, fout) else: cmd = seqret + " fastq-{0}::{1} fastq-{2}::stdout".\ format(fin, infastq, fout) sh(cmd, outfile=outfastq) return outfastq
def gatk(args): """ %prog gatk bamfile reference.fasta Call SNPs based on GATK best practices. """ p = OptionParser(gatk.__doc__) p.add_option("--indelrealign", default=False, action="store_true", help="Perform indel realignment") p.set_home("gatk") p.set_home("picard") p.set_phred() p.set_cpus(cpus=24) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, ref = args pf = bamfile.rsplit(".", 1)[0] mm = MakeManager() picard = "java -Xmx32g -jar {0}/picard.jar".format(opts.picard_home) tk = "java -Xmx32g -jar {0}/GenomeAnalysisTK.jar".format(opts.gatk_home) tk += " -R {0}".format(ref) # Step 0 - build reference dictfile = ref.rsplit(".", 1)[0] + ".dict" cmd1 = picard + " CreateSequenceDictionary" cmd1 += " R={0} O={1}".format(ref, dictfile) cmd2 = "samtools faidx {0}".format(ref) mm.add(ref, dictfile, (cmd1, cmd2)) # Step 1 - sort bam sortedbamfile = pf + ".sorted.bam" cmd = picard + " SortSam" cmd += " INPUT={0} OUTPUT={1}".format(bamfile, sortedbamfile) cmd += " SORT_ORDER=coordinate CREATE_INDEX=true" mm.add(bamfile, sortedbamfile, cmd) # Step 2 - mark duplicates dedupbamfile = pf + ".dedup.bam" cmd = picard + " MarkDuplicates" cmd += " INPUT={0} OUTPUT={1}".format(sortedbamfile, dedupbamfile) cmd += " METRICS_FILE=dedup.log CREATE_INDEX=true" mm.add(sortedbamfile, dedupbamfile, cmd) if opts.indelrealign: # Step 3 - create indel realignment targets intervals = pf + ".intervals" cmd = tk + " -T RealignerTargetCreator" cmd += " -I {0} -o {1}".format(dedupbamfile, intervals) mm.add(dedupbamfile, intervals, cmd) # Step 4 - indel realignment realignedbamfile = pf + ".realigned.bam" cmd = tk + " -T IndelRealigner" cmd += " -targetIntervals {0}".format(intervals) cmd += " -I {0} -o {1}".format(dedupbamfile, realignedbamfile) mm.add((dictfile, intervals), realignedbamfile, cmd) else: realignedbamfile = dedupbamfile # Step 5 - SNP calling vcf = pf + ".vcf" cmd = tk + " -T HaplotypeCaller" cmd += " -I {0}".format(realignedbamfile) cmd += " --genotyping_mode DISCOVERY" cmd += " -stand_emit_conf 10 -stand_call_conf 30" cmd += " -nct {0}".format(opts.cpus) cmd += " -o {0}".format(vcf) if opts.phred == "64": cmd += " --fix_misencoded_quality_scores" mm.add(realignedbamfile, vcf, cmd) # Step 6 - SNP filtering filtered_vcf = pf + ".filtered.vcf" cmd = tk + " -T VariantFiltration" cmd += " -V {0}".format(vcf) cmd += ' --filterExpression "DP < 10 || DP > 300 || QD < 2.0 || FS > 60.0 || MQ < 40.0"' cmd += ' --filterName "LOWQUAL"' cmd += ' --genotypeFilterExpression "isHomVar == 1"' cmd += ' --genotypeFilterName "HOMOVAR"' cmd += ' --genotypeFilterExpression "isHet == 1"' cmd += ' --genotypeFilterName "HET"' cmd += " -o {0}".format(filtered_vcf) mm.add(vcf, filtered_vcf, cmd) mm.write()
def tophat(args): """ %prog tophat folder reference Run tophat on a folder of reads. """ from jcvi.apps.bowtie import check_index from jcvi.formats.fastq import guessoffset p = OptionParser(tophat.__doc__) p.add_option("--gtf", help="Reference annotation [default: %default]") p.add_option("--single", default=False, action="store_true", help="Single end mapping") p.add_option("--intron", default=15000, type="int", help="Max intron size [default: %default]") p.add_option("--dist", default=-50, type="int", help="Mate inner distance [default: %default]") p.add_option("--stdev", default=50, type="int", help="Mate standard deviation [default: %default]") p.set_phred() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) num = 1 if opts.single else 2 folder, reference = args reference = check_index(reference) for p, prefix in iter_project(folder, n=num): outdir = "{0}_tophat".format(prefix) outfile = op.join(outdir, "accepted_hits.bam") if op.exists(outfile): logging.debug("File `{0}` found. Skipping.".format(outfile)) continue cmd = "tophat -p {0}".format(opts.cpus) if opts.gtf: cmd += " -G {0}".format(opts.gtf) cmd += " -o {0}".format(outdir) if num == 1: # Single-end a, = p else: # Paired-end a, b = p cmd += " --max-intron-length {0}".format(opts.intron) cmd += " --mate-inner-dist {0}".format(opts.dist) cmd += " --mate-std-dev {0}".format(opts.stdev) phred = opts.phred or str(guessoffset([a])) if phred == "64": cmd += " --phred64-quals" cmd += " {0} {1}".format(reference, " ".join(p)) sh(cmd)
def gatk(args): """ %prog gatk bamfile reference.fasta Call SNPs based on GATK best practices. """ p = OptionParser(gatk.__doc__) p.add_option("--indelrealign", default=False, action="store_true", help="Perform indel realignment") p.set_home("gatk") p.set_home("picard") p.set_phred() p.set_cpus(cpus=24) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) bamfile, ref = args pf = bamfile.rsplit(".", 1)[0] mm = MakeManager() picard = "java -Xmx32g -jar {0}/picard.jar".format(opts.picard_home) tk = "java -Xmx32g -jar {0}/GenomeAnalysisTK.jar".format(opts.gatk_home) tk += " -R {0}".format(ref) # Step 0 - build reference dictfile = ref.rsplit(".", 1)[0] + ".dict" cmd1 = picard + " CreateSequenceDictionary" cmd1 += " R={0} O={1}".format(ref, dictfile) cmd2 = "samtools faidx {0}".format(ref) mm.add(ref, dictfile, (cmd1, cmd2)) # Step 1 - sort bam sortedbamfile = pf + ".sorted.bam" cmd = picard + " SortSam" cmd += " INPUT={0} OUTPUT={1}".format(bamfile, sortedbamfile) cmd += " SORT_ORDER=coordinate CREATE_INDEX=true" mm.add(bamfile, sortedbamfile, cmd) # Step 2 - mark duplicates dedupbamfile = pf + ".dedup.bam" cmd = picard + " MarkDuplicates" cmd += " INPUT={0} OUTPUT={1}".format(sortedbamfile, dedupbamfile) cmd += " METRICS_FILE=dedup.log CREATE_INDEX=true" mm.add(sortedbamfile, dedupbamfile, cmd) if opts.indelrealign: # Step 3 - create indel realignment targets intervals = pf + ".intervals" cmd = tk + " -T RealignerTargetCreator" cmd += " -I {0} -o {1}".format(dedupbamfile, intervals) mm.add(dedupbamfile, intervals, cmd) # Step 4 - indel realignment realignedbamfile = pf + ".realigned.bam" cmd = tk + " -T IndelRealigner" cmd += " -targetIntervals {0}".format(intervals) cmd += " -I {0} -o {1}".format(dedupbamfile, realignedbamfile) mm.add((dictfile, intervals), realignedbamfile, cmd) else: realignedbamfile = dedupbamfile # Step 5 - SNP calling vcf = pf + ".vcf" cmd = tk + " -T HaplotypeCaller" cmd += " -I {0}".format(realignedbamfile) cmd += " --genotyping_mode DISCOVERY" cmd += " -stand_emit_conf 10 -stand_call_conf 30" cmd += " -nct {0}".format(opts.cpus) cmd += " -o {0}".format(vcf) if opts.phred == "64": cmd += " --fix_misencoded_quality_scores" mm.add(realignedbamfile, vcf, cmd) # Step 6 - SNP filtering filtered_vcf = pf + ".filtered.vcf" cmd = tk + " -T VariantFiltration" cmd += " -V {0}".format(vcf) cmd += ' --filterExpression "DP < 10 || DP > 300 || QD < 2.0 || FS > 60.0 || MQ < 40.0"' cmd += ' --filterName "LOWQUAL"' cmd += ' --genotypeFilterExpression "isHomVar == 1"' cmd += ' --genotypeFilterName "HOMOVAR"' cmd += ' --genotypeFilterExpression "isHet == 1"' cmd += ' --genotypeFilterName "HET"' cmd += " -o {0}".format(filtered_vcf) mm.add(vcf, filtered_vcf, cmd) mm.write()
def trim(args): """ %prog trim fastqfiles Trim reads using TRIMMOMATIC. If two fastqfiles are given, then it invokes the paired reads mode. See manual: <http://www.usadellab.org/cms/index.php?page=trimmomatic> """ tv = "0.32" TrimJar = "trimmomatic-{0}.jar".format(tv) p = OptionParser(trim.__doc__) p.add_option("--path", default=op.join("~/bin", TrimJar), help="Path to trimmomatic jar file [default: %default]") p.set_phred() p.add_option("--nofrags", default=False, action="store_true", help="Discard frags file in PE mode [default: %default]") p.add_option("--minqv", default=15, type="int", help="Average qv after trimming [default: %default]") p.add_option("--minlen", default=36, type="int", help="Minimum length after trimming [default: %default]") p.add_option("--adapteronly", default=False, action="store_true", help="Only trim adapters with no qv trimming [default: %default]") p.add_option("--nogz", default=False, action="store_true", help="Do not write to gzipped files [default: %default]") p.add_option("--log", default=None, dest="trimlog", help="Specify a `trimlog` file [default: %default]") p.set_cpus(cpus=4) opts, args = p.parse_args(args) if len(args) not in (1, 2): sys.exit(not p.print_help()) path = op.expanduser(opts.path) url = \ "http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/Trimmomatic-{0}.zip"\ .format(tv) if not op.exists(path): path = download(url) TrimUnzipped = "Trimmomatic-" + tv if not op.exists(TrimUnzipped): sh("unzip " + path) os.remove(path) path = op.join(TrimUnzipped, TrimJar) assert op.exists(path), \ "Couldn't find Trimmomatic jar file at `{0}`".\ format(path) adaptersfile = "adapters.fasta" Adapters = must_open(op.join(datadir, adaptersfile)).read() write_file(adaptersfile, Adapters, skipcheck=True) assert op.exists(adaptersfile), \ "Please place the illumina adapter sequence in `{0}`".\ format(adaptersfile) if opts.phred is None: offset = guessoffset([args[0]]) else: offset = int(opts.phred) phredflag = " -phred{0}".format(offset) threadsflag = " -threads {0}".format(opts.cpus) if opts.trimlog: trimlog = " -trimlog {0}".format(opts.trimlog) cmd = "java -Xmx4g -jar {0}".format(path) frags = ".frags.fastq" pairs = ".pairs.fastq" if not opts.nogz: frags += ".gz" pairs += ".gz" get_prefix = lambda x: op.basename(x).replace(".gz", "").rsplit(".", 1)[0] get_dirname = lambda x: "{0}/".format(op.dirname(x)) if op.dirname(x) else '' if len(args) == 1: cmd += " SE" cmd += phredflag cmd += threadsflag if opts.trimlog: cmd += trimlog fastqfile, = args prefix = get_prefix(fastqfile) dirname = get_dirname(fastqfile) frags1 = dirname + prefix + frags cmd += " {0}".format(" ".join((fastqfile, frags1))) else: cmd += " PE" cmd += phredflag cmd += threadsflag if opts.trimlog: cmd += trimlog fastqfile1, fastqfile2 = args prefix1 = get_prefix(fastqfile1) dirname1 = get_dirname(fastqfile1) prefix2 = get_prefix(fastqfile2) dirname2 = get_dirname(fastqfile2) pairs1 = dirname1 + prefix1 + pairs pairs2 = dirname2 + prefix2 + pairs frags1 = dirname1 + prefix1 + frags frags2 = dirname2 + prefix2 + frags if opts.nofrags: frags1 = "/dev/null" frags2 = "/dev/null" cmd += " {0}".format(" ".join((fastqfile1, fastqfile2, \ pairs1, frags1, pairs2, frags2))) cmd += " ILLUMINACLIP:{0}:2:30:10".format(adaptersfile) if not opts.adapteronly: cmd += " LEADING:3 TRAILING:3" cmd += " SLIDINGWINDOW:4:{0}".format(opts.minqv) cmd += " MINLEN:{0}".format(opts.minlen) if offset != 33: cmd += " TOPHRED33" sh(cmd)