def star(args): """ %prog star folder reference Run star on a folder with reads. """ p = OptionParser(star.__doc__) p.add_option("--single", default=False, action="store_true", help="Single end mapping") p.set_fastq_names() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) folder, reference = args cpus = opts.cpus mm = MakeManager() num = 1 if opts.single else 2 folder, reference = args gd = "GenomeDir" mkdir(gd) STAR = "STAR --runThreadN {0} --genomeDir {1}".format(cpus, gd) # Step 0: build genome index genomeidx = op.join(gd, "Genome") if need_update(reference, genomeidx): cmd = STAR + " --runMode genomeGenerate" cmd += " --genomeFastaFiles {0}".format(reference) mm.add(reference, genomeidx, cmd) # Step 1: align for p, prefix in iter_project(folder, opts.names, num): pf = "{0}_star".format(prefix) bamfile = pf + "Aligned.sortedByCoord.out.bam" cmd = STAR + " --readFilesIn {0}".format(" ".join(p)) if p[0].endswith(".gz"): cmd += " --readFilesCommand zcat" cmd += " --outSAMtype BAM SortedByCoordinate" cmd += " --outFileNamePrefix {0}".format(pf) cmd += " --twopassMode Basic" # Compatibility for cufflinks cmd += " --outSAMstrandField intronMotif" cmd += " --outFilterIntronMotifs RemoveNoncanonical" mm.add(p, bamfile, cmd) mm.write()
def star(args): """ %prog star folder reference Run star on a folder with reads. """ p = OptionParser(star.__doc__) p.add_option("--single", default=False, action="store_true", help="Single end mapping") p.set_fastq_names() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) folder, reference = args cpus = opts.cpus mm = MakeManager() num = 1 if opts.single else 2 folder, reference = args gd = "GenomeDir" mkdir(gd) STAR = "STAR --runThreadN {0} --genomeDir {1}".format(cpus, gd) # Step 0: build genome index genomeidx = op.join(gd, "Genome") if need_update(reference, genomeidx): cmd = STAR + " --runMode genomeGenerate" cmd += " --genomeFastaFiles {0}".format(reference) mm.add(reference, genomeidx, cmd) # Step 1: align for p, prefix in iter_project(folder, opts.names, num): pf = "{0}_star".format(prefix) bamfile = pf + "Aligned.sortedByCoord.out.bam" cmd = STAR + " --readFilesIn {0}".format(" ".join(p)) if p[0].endswith(".gz"): cmd += " --readFilesCommand zcat" cmd += " --outSAMtype BAM SortedByCoordinate" cmd += " --outFileNamePrefix {0}".format(pf) cmd += " --twopassMode Basic" # Compatibility for cufflinks cmd += " --outSAMstrandField intronMotif" cmd += " --outFilterIntronMotifs RemoveNoncanonical" mm.add(p, bamfile, cmd) mm.write()
def snpflow(args): """ %prog snpflow trimmed reference.fasta Run SNP calling pipeline until allele_counts are generated. This includes generation of native files, SNP_Het file. Speedup for fragmented genomes are also supported. """ p = OptionParser(snpflow.__doc__) p.set_fastq_names() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) trimmed, ref = args nseqs = len(Fasta(ref)) supercat = nseqs >= 1000 if supercat: logging.debug("Total seqs in ref: {0} (supercat={1})".\ format(nseqs, supercat)) reads, samples = scan_read_files(trimmed, opts.names) # Set up directory structure nativedir, countsdir = "native", "allele_counts" for d in (nativedir, countsdir): mkdir(d) mm = MakeManager() # Step 0 - index database db = op.join(*check_index(ref, supercat=supercat, go=False)) cmd = "python -m jcvi.apps.gmap index {0}".format(ref) if supercat: cmd += " --supercat" coordsfile = db + ".coords" supercatfile = ref.rsplit(".", 1)[0] + ".supercat.fasta" mm.add(ref, (db, coordsfile), cmd) else: mm.add(ref, db, cmd) # Step 1 - GSNAP alignment and conversion to native file allnatives = [] allsamstats = [] gmapdb = supercatfile if supercat else ref for f in reads: prefix = get_prefix(f, ref) gsnapfile = op.join(nativedir, prefix + ".gsnap") nativefile = op.join(nativedir, prefix + ".unique.native") samstatsfile = op.join(nativedir, prefix + ".unique.sam.stats") cmd = "python -m jcvi.apps.gmap align {0} {1}".format(gmapdb, f) cmd += " --outdir={0} --native --cpus=1".format(nativedir) mm.add((f, db), nativefile, cmd) cmd = "python -m jcvi.apps.gmap bam {0} {1} --cpus=1".\ format(gsnapfile, gmapdb) mm.add(nativefile, samstatsfile, cmd) allnatives.append(nativefile) allsamstats.append(samstatsfile) # Step 2 - call SNP discovery if supercat: nativeconverted = nativedir + "-converted" mkdir(nativeconverted) allnativesc = [op.join(nativeconverted, op.basename(x)) for x in allnatives] cmd = "tGBS-Convert_Pseudo_Genome_NATIVE_Coordinates.pl" cmd += " -i {0}/*.native -o {1}".format(nativedir, nativeconverted) cmd += " -c {0}".format(coordsfile) cmds = ["rm -rf {0}".format(nativeconverted), cmd] mm.add(allnatives + [coordsfile], allnativesc, cmds) runfile = "speedup.sh" write_file(runfile, speedupsh.format(nativeconverted, opts.cpus)) nativedir = nativeconverted allsnps = [op.join(nativedir, "{0}.SNPs_Het.txt".format(x)) for x in samples] mm.add(allnativesc, allsnps, "./{0}".format(runfile)) else: for s in samples: snpfile = op.join(nativedir, "{0}.SNPs_Het.txt".format(s)) cmd = "SNP_Discovery-short.pl" cmd += " -native {0}/{1}.*unique.native".format(nativedir, s) cmd += " -o {0} -a 2 -ac 0.3 -c 0.8".format(snpfile) flist = [x for x in allnatives if op.basename(x).split(".")[0] == s] mm.add(flist, snpfile, cmd) # Step 3 - generate equal file allsnps = [op.join(nativedir, "{0}.SNPs_Het.txt".format(x)) for x in samples] for s in samples: equalfile = op.join(nativedir, "{0}.equal".format(s)) cmd = "extract_reference_alleles.pl" cmd += " --native {0}/{1}.*unique.native".format(nativedir, s) cmd += " --genotype {0}/{1}.SNPs_Het.txt".format(nativedir, s) cmd += " --allgenotypes {0}/*.SNPs_Het.txt".format(nativedir) cmd += " --fasta {0} --output {1}".format(ref, equalfile) mm.add(allsnps, equalfile, cmd) # Step 4 - generate snp matrix allequals = [op.join(nativedir, "{0}.equal".format(x)) for x in samples] matrix = "snps.matrix.txt" cmd = "generate_matrix.pl" cmd += " --tables {0}/*SNPs_Het.txt --equal {0}/*equal".format(nativedir) cmd += " --fasta {0} --output {1}".format(ref, matrix) mm.add(allsnps + allequals, matrix, cmd) # Step 5 - generate allele counts allcounts = [] for s in samples: allele_counts = op.join(countsdir, "{0}.SNPs_Het.allele_counts".format(s)) cmd = "count_reads_per_allele.pl -m snps.matrix.txt" cmd += " -s {0} --native {1}/{0}.*unique.native".format(s, nativedir) cmd += " -o {0}".format(allele_counts) mm.add(matrix, allele_counts, cmd) allcounts.append(allele_counts) # Step 6 - generate raw snps rawsnps = "Genotyping.H3.txt" cmd = "/home/shared/scripts/delin/SamplesGenotyping.pl --h**o 3" cmd += " -pf allele_counts -f {0} --outfile {1}".format(countsdir, rawsnps) cmds = ["rm -f {0}".format(rawsnps), cmd] mm.add(allcounts, rawsnps, cmds) # Step 7 - generate alignment report sam_summary = "sam.summary" cmd = "/home/shared/scripts/eddyyeh/alignment_stats.pl" cmd += " -f {0} -o {1}".format(" ".join(allsamstats), sam_summary) mm.add(allsamstats, sam_summary, cmd) native_summary = "native.summary" cmd = "/home/shared/scripts/eddyyeh/alignment_stats.pl" cmd += " -n {0} -o {1}".format(" ".join(allnatives), native_summary) mm.add(allnatives, native_summary, cmd) mm.write()
def novo2(args): """ %prog novo2 trimmed projectname Reference-free tGBS pipeline v2. """ p = OptionParser(novo2.__doc__) p.set_fastq_names() p.set_align(pctid=94) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) trimmed, pf = args pctid = opts.pctid reads, samples = scan_read_files(trimmed, opts.names) # Set up directory structure clustdir = "uclust" acdir ="allele_counts" for d in (clustdir, acdir): mkdir(d) mm = MakeManager() clustfiles = [] # Step 0 - clustering within sample for s in samples: flist = [x for x in reads if op.basename(x).split(".")[0] == s] outfile = s + ".P{0}.clustS".format(pctid) outfile = op.join(clustdir, outfile) cmd = "python -m jcvi.apps.uclust cluster --cpus=8" cmd += " {0} {1}".format(s, " ".join(flist)) cmd += " --outdir={0}".format(clustdir) cmd += " --pctid={0}".format(pctid) mm.add(flist, outfile, cmd) clustfiles.append(outfile) # Step 1 - make consensus within sample allcons = [] for s, clustfile in zip(samples, clustfiles): outfile = s + ".P{0}.consensus".format(pctid) outfile = op.join(clustdir, outfile) cmd = "python -m jcvi.apps.uclust consensus" cmd += " {0}".format(clustfile) mm.add(clustfile, outfile, cmd) allcons.append(outfile) # Step 2 - clustering across samples clustSfile = pf + ".P{0}.clustS".format(pctid) cmd = "python -m jcvi.apps.uclust mcluster {0}".format(" ".join(allcons)) cmd += " --prefix={0}".format(pf) mm.add(allcons, clustSfile, cmd) # Step 3 - make consensus across samples locifile = pf + ".P{0}.loci".format(pctid) cmd = "python -m jcvi.apps.uclust mconsensus {0}".format(" ".join(allcons)) cmd += " --prefix={0}".format(pf) mm.add(allcons + [clustSfile], locifile, cmd) mm.write()
def prepare(args): """ %prog prepare [--options] folder [--bam rnaseq.coordSorted.bam] Run Trinity on a folder of reads. When paired-end (--paired) mode is on, filenames will be scanned based on whether they contain the patterns ("_1_" and "_2_") or (".1." and ".2.") or ("_1." and "_2."). By default, prepare script for DN-Trinity. If coord-sorted BAM is provided, prepare script for GG-Trinity, using BAM as starting point. Newer versions of trinity can take multiple fastq files as input. If "--merge" is specified, the fastq files are merged together before assembling """ p = OptionParser(prepare.__doc__) p.add_option("--paired", default=False, action="store_true", help="Paired-end mode [default: %default]") p.add_option("--merge", default=False, action="store_true", help="Merge individual input fastq's into left/right/single" + \ " file(s) [default: %default]") p.set_trinity_opts() p.set_fastq_names() p.set_grid() opts, args = p.parse_args(args) if len(args) not in (1, 2): sys.exit(not p.print_help()) inparam, = args[:1] paired = opts.paired merge = opts.merge trinity_home = opts.trinity_home hpc_grid_runner_home = opts.hpcgridrunner_home method = "DN" bam = opts.bam if bam and op.exists(bam): bam = op.abspath(bam) method = "GG" pf = inparam.split(".")[0] tfolder = "{0}_{1}".format(pf, method) cwd = os.getcwd() mkdir(tfolder) os.chdir(tfolder) cmds = [] # set TRINITY_HOME env variable when preparing shell script env_cmd = 'export TRINITY_HOME="{0}"'.format(trinity_home) cmds.append(env_cmd) if method == "DN": assert op.exists("../" + inparam) flist = iglob("../" + inparam, opts.names) if paired: f1 = [x for x in flist if "_1_" in x or ".1." in x or "_1." in x or "_R1" in x] f2 = [x for x in flist if "_2_" in x or ".2." in x or "_2." in x or "_R2" in x] assert len(f1) == len(f2) if merge: r1, r2 = "left.fastq", "right.fastq" reads = ((f1, r1), (f2, r2)) else: if merge: r = "single.fastq" reads = ((flist, r), ) if merge: for fl, r in reads: fm = FileMerger(fl, r) fm.merge(checkexists=True) cmd = op.join(trinity_home, "Trinity") cmd += " --seqType fq --max_memory {0} --CPU {1}".format(opts.max_memory, opts.cpus) cmd += " --min_contig_length {0}".format(opts.min_contig_length) if opts.bflyGCThreads: cmd += " --bflyGCThreads {0}".format(opts.bflyGCThreads) if method == "GG": cmd += " --genome_guided_bam {0}".format(bam) cmd += " --genome_guided_max_intron {0}".format(opts.max_intron) else: if paired: if merge: cmd += " --left {0} --right {1}".format(reads[0][-1], reads[1][-1]) else: cmd += " --left {0}".format(",".join(f1)) cmd += " --right {0}".format(",".join(f2)) else: if merge: cmd += " --single {0}".format(reads[0][-1]) else: for f in flist: cmd += " --single {0}".format(f) if opts.grid and opts.grid_conf_file: hpc_grid_runner = op.join(hpc_grid_runner_home, "hpc_cmds_GridRunner.pl") hpc_grid_conf_file = op.join(hpc_grid_runner_home, "hpc_conf", opts.grid_conf_file) assert op.exists(hpc_grid_conf_file), "HpcGridRunner conf file does not exist: {0}".format(hpc_grid_conf_file) cmd += ' --grid_exec "{0} --grid_conf {1} -c"'.format(hpc_grid_runner, hpc_grid_conf_file) if opts.extra: cmd += " {0}".format(opts.extra) cmds.append(cmd) if opts.cleanup: cleanup_cmd = 'rm -rf !("Trinity.fasta"|"Trinity.gene_trans_map"|"Trinity.timing")' \ if method == "DN" else \ 'rm -rf !("Trinity-GG.fasta"|"Trinity-GG.gene_trans_map"|"Trinity.timing")' cmd.append(cleanup_cmd) runfile = "run.sh" write_file(runfile, "\n".join(cmds)) os.chdir(cwd)
def snpflow(args): """ %prog snpflow trimmed reference.fasta Run SNP calling pipeline until allele_counts are generated. This includes generation of native files, SNP_Het file. Speedup for fragmented genomes are also supported. """ p = OptionParser(snpflow.__doc__) p.set_fastq_names() p.set_cpus() opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) trimmed, ref = args nseqs = len(Fasta(ref)) supercat = nseqs >= 1000 if supercat: logging.debug("Total seqs in ref: {0} (supercat={1})".\ format(nseqs, supercat)) reads, samples = scan_read_files(trimmed, opts.names) # Set up directory structure nativedir, countsdir = "native", "allele_counts" for d in (nativedir, countsdir): mkdir(d) mm = MakeManager() # Step 0 - index database db = op.join(*check_index(ref, supercat=supercat, go=False)) cmd = "python -m jcvi.apps.gmap index {0}".format(ref) if supercat: cmd += " --supercat" coordsfile = db + ".coords" supercatfile = ref.rsplit(".", 1)[0] + ".supercat.fasta" mm.add(ref, (db, coordsfile), cmd) else: mm.add(ref, db, cmd) # Step 1 - GSNAP alignment and conversion to native file allnatives = [] allsamstats = [] gmapdb = supercatfile if supercat else ref for f in reads: prefix = get_prefix(f, ref) gsnapfile = op.join(nativedir, prefix + ".gsnap") nativefile = op.join(nativedir, prefix + ".unique.native") samstatsfile = op.join(nativedir, prefix + ".unique.sam.stats") cmd = "python -m jcvi.apps.gmap align {0} {1}".format(gmapdb, f) cmd += " --outdir={0} --native --cpus=1".format(nativedir) mm.add((f, db), nativefile, cmd) cmd = "python -m jcvi.apps.gmap bam {0} {1} --cpus=1".\ format(gsnapfile, gmapdb) mm.add(nativefile, samstatsfile, cmd) allnatives.append(nativefile) allsamstats.append(samstatsfile) # Step 2 - call SNP discovery if supercat: nativeconverted = nativedir + "-converted" mkdir(nativeconverted) allnativesc = [op.join(nativeconverted, op.basename(x)) for x in allnatives] cmd = "tGBS-Convert_Pseudo_Genome_NATIVE_Coordinates.pl" cmd += " -i {0}/*.native -o {1}".format(nativedir, nativeconverted) cmd += " -c {0}".format(coordsfile) cmds = ["rm -rf {0}".format(nativeconverted), cmd] mm.add(allnatives + [coordsfile], allnativesc, cmds) runfile = "speedup.sh" write_file(runfile, speedupsh.format(nativeconverted, opts.cpus)) nativedir = nativeconverted allsnps = [op.join(nativedir, "{0}.SNPs_Het.txt".format(x)) for x in samples] mm.add(allnativesc, allsnps, "./{0}".format(runfile)) else: for s in samples: snpfile = op.join(nativedir, "{0}.SNPs_Het.txt".format(s)) cmd = "SNP_Discovery-short.pl" cmd += " -native {0}/{1}.*unique.native".format(nativedir, s) cmd += " -o {0} -a 2 -ac 0.3 -c 0.8".format(snpfile) flist = [x for x in allnatives if op.basename(x).split(".")[0] == s] mm.add(flist, snpfile, cmd) # Step 3 - generate equal file allsnps = [op.join(nativedir, "{0}.SNPs_Het.txt".format(x)) for x in samples] for s in samples: equalfile = op.join(nativedir, "{0}.equal".format(s)) cmd = "extract_reference_alleles.pl" cmd += " --native {0}/{1}.*unique.native".format(nativedir, s) cmd += " --genotype {0}/{1}.SNPs_Het.txt".format(nativedir, s) cmd += " --allgenotypes {0}/*.SNPs_Het.txt".format(nativedir) cmd += " --fasta {0} --output {1}".format(ref, equalfile) mm.add(allsnps, equalfile, cmd) # Step 4 - generate snp matrix allequals = [op.join(nativedir, "{0}.equal".format(x)) for x in samples] matrix = "snps.matrix.txt" cmd = "generate_matrix.pl" cmd += " --tables {0}/*SNPs_Het.txt --equal {0}/*equal".format(nativedir) cmd += " --fasta {0} --output {1}".format(ref, matrix) mm.add(allsnps + allequals, matrix, cmd) # Step 5 - generate allele counts allcounts = [] for s in samples: allele_counts = op.join(countsdir, "{0}.SNPs_Het.allele_counts".format(s)) cmd = "count_reads_per_allele.pl -m snps.matrix.txt" cmd += " -s {0} --native {1}/{0}.*unique.native".format(s, nativedir) cmd += " -o {0}".format(allele_counts) mm.add(matrix, allele_counts, cmd) allcounts.append(allele_counts) # Step 6 - generate raw snps rawsnps = "Genotyping.H3.txt" cmd = "/home/shared/scripts/delin/SamplesGenotyping.pl --h**o 3" cmd += " -pf allele_counts -f {0} --outfile {1}".format(countsdir, rawsnps) cmds = ["rm -f {0}".format(rawsnps), cmd] mm.add(allcounts, rawsnps, cmds) # Step 7 - generate alignment report sam_summary = "sam.summary" cmd = "/home/shared/scripts/eddyyeh/alignment_stats.pl" cmd += " -f {0} -o {1}".format(" ".join(allsamstats), sam_summary) mm.add(allsamstats, sam_summary, cmd) native_summary = "native.summary" cmd = "/home/shared/scripts/eddyyeh/alignment_stats.pl" cmd += " -n {0} -o {1}".format(" ".join(allnatives), native_summary) mm.add(allnatives, native_summary, cmd) mm.write()
def novo2(args): """ %prog novo2 trimmed projectname Reference-free tGBS pipeline v2. """ p = OptionParser(novo2.__doc__) p.set_fastq_names() p.set_align(pctid=95) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) trimmed, pf = args pctid = opts.pctid reads, samples = scan_read_files(trimmed, opts.names) # Set up directory structure clustdir = "uclust" acdir ="allele_counts" for d in (clustdir, acdir): mkdir(d) mm = MakeManager() clustfiles = [] # Step 0 - clustering within sample for s in samples: flist = [x for x in reads if op.basename(x).split(".")[0] == s] outfile = s + ".P{0}.clustS".format(pctid) outfile = op.join(clustdir, outfile) cmd = "python -m jcvi.apps.uclust cluster --cpus=8" cmd += " {0} {1}".format(s, " ".join(flist)) cmd += " --outdir={0}".format(clustdir) cmd += " --pctid={0}".format(pctid) mm.add(flist, outfile, cmd) clustfiles.append(outfile) # Step 1 - make consensus within sample allcons = [] for s, clustfile in zip(samples, clustfiles): outfile = s + ".P{0}.consensus".format(pctid) outfile = op.join(clustdir, outfile) cmd = "python -m jcvi.apps.uclust consensus" cmd += " {0}".format(clustfile) mm.add(clustfile, outfile, cmd) allcons.append(outfile) # Step 2 - clustering across samples clustSfile = pf + ".P{0}.clustS".format(pctid) cmd = "python -m jcvi.apps.uclust mcluster {0}".format(" ".join(allcons)) cmd += " --prefix={0}".format(pf) mm.add(allcons, clustSfile, cmd) # Step 3 - make consensus across samples locifile = pf + ".P{0}.loci".format(pctid) cmd = "python -m jcvi.apps.uclust mconsensus {0}".format(" ".join(allcons)) cmd += " --prefix={0}".format(pf) mm.add(allcons + [clustSfile], locifile, cmd) mm.write()