def clean_fasta(args):
dirw = make_genomedir(args.species)
os.chdir(dirw)
for fname in ["raw.fix.fas.index", "11_genome.fas.index"]:
if op.isfile(fname):
os.remove(fname)
if op.islink("10_genome.fna"): os.unlink("10_genome.fna")
if op.isfile("10_genome.fna") and not args.overwrite:
logging.debug("10_genome.fna already exits: skipped")
elif op.isfile("08_seq_map/renamed.fna"):
sh("ln -sf 08_seq_map/renamed.fna 10_genome.fna")
if op.isfile("08_seq_map/renamed.sizes"):
sh("ln -sf 08_seq_map/renamed.sizes 10_genome.sizes")
else:
logging.error("08_seq_map/renamed.fna not there")
sys.exit(1)
if not op.isdir("15_intervals"):
mkdir("15_intervals")
if op.isfile("15_intervals/01.chrom.bed") and not args.overwrite:
logging.debug("01.chrom.bed already exits - skipped")
else:
sh("fasta size --bed 10_genome.fna > 15_intervals/01.chrom.bed")
if op.isfile("15_intervals/01.chrom.sizes") and not args.overwrite:
logging.debug("01.chrom.sizes already exits - skipped")
else:
sh("faSize -detailed 10_genome.fna > 15_intervals/01.chrom.sizes")
if op.isfile("15_intervals/11.gap.bed") and not args.overwrite:
logging.debug("11.gap.bed already exits - skipped")
else:
sh("fasta gaps 10_genome.fna > 15_intervals/11.gap.bed")
def clean_fasta(args):
dirw = make_genomedir(args.species)
os.chdir(dirw)
for fname in ["raw.fix.fas.index", "11_genome.fas.index"]:
if op.isfile(fname):
os.remove(fname)
if op.islink("10_genome.fna"): os.unlink("10_genome.fna")
if op.isfile("10_genome.fna") and not args.overwrite:
logging.debug("10_genome.fna already exits: skipped")
elif op.isfile("08_seq_map/renamed.fna"):
sh("ln -sf 08_seq_map/renamed.fna 10_genome.fna")
if op.isfile("08_seq_map/renamed.sizes"):
sh("ln -sf 08_seq_map/renamed.sizes 10_genome.sizes")
else:
logging.error("08_seq_map/renamed.fna not there")
sys.exit(1)
if not op.isdir("15_intervals"):
mkdir("15_intervals")
if op.isfile("15_intervals/01.chrom.bed") and not args.overwrite:
logging.debug("01.chrom.bed already exits - skipped")
else:
sh("fasta size --bed 10_genome.fna > 15_intervals/01.chrom.bed")
if op.isfile("15_intervals/01.chrom.sizes") and not args.overwrite:
logging.debug("01.chrom.sizes already exits - skipped")
else:
sh("faSize -detailed 10_genome.fna > 15_intervals/01.chrom.sizes")
if op.isfile("15_intervals/11.gap.bed") and not args.overwrite:
logging.debug("11.gap.bed already exits - skipped")
else:
sh("fasta gaps 10_genome.fna > 15_intervals/11.gap.bed")
def check_cfg_fqtrim(c, njob = 1, noutdir = 3):
c.outdirs = c.outdir.split(",")
assert len(c.outdirs) == noutdir, "not %s outdirs: %s" % (noutdir, c.outdir)
for subdir in [c.dirw, c.temp_dir] + c.outdirs:
if not op.isdir(subdir):
mkdir(subdir)
for fn in [c.ilist, c.adapter, c.trimmomatic]:
assert op.isfile(fn), "cannot read %s" % fn
for key in ['fastqc', 'parallel']:
fp = which(c[key])
assert fp is not None, "not executable: %s" % c[key]
c[key] = fp
c.paired = str2bool(c.paired)
c.pbs_walltimes = c.pbs_walltime.split(",")
c.pbs_ppns = c.pbs_ppn.split(",")
c.pbs_queues = c.pbs_queue.split(",")
assert njob == len(c.pbs_queues) == len(c.pbs_walltimes) == len(c.pbs_ppns), "not %d jobs: %s" % (njob, c.pbs_queue)
c.njob = njob
return c
def sra(args):
"""
%prog sra [term|term.ids]
Given an SRA run ID, fetch the corresponding .sra file from the sra-instant FTP.
The term can also be a file containing list of SRR ids, one per line.
Once downloaded, the SRA file is processed through `fastq-dump` to produce
FASTQ formatted sequence files, which are gzipped by default.
"""
p = OptionParser(sra.__doc__)
sp1.add_argument("--nogzip", dest="nogzip",
default=False, action="store_true",
help="Do not gzip the FASTQ generated by fastq-dump")
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(not p.print_help())
term, = args
if op.isfile(term):
terms = [x.strip() for x in open(term)]
else:
terms = [term]
for term in terms:
srafile = download_srr_term(term)
pf = srafile.split(".")[0]
mkdir(pf)
_opts = [srafile, "--paired", "--outdir={0}".format(pf)]
if not args.nogzip:
_args.append("--compress=gzip")
fromsra(_opts)
def index(cfg, args):
c = AttrDict(cfg['index'])
c = check_cfg_index(c)
if args.check:
return 0
os.chdir(c.dirw)
jcmds = [[
"cd %s" % c.dirw
],[
"cd %s" % c.dirw
],[
"cd %s" % c.dirw
]]
bcfgs = [
[dict(opt = 'bash')],
[dict(opt = 'parallel', thread = c.pbs_ppns[1])],
[dict(opt = 'bash')]
]
assert c.njob == len(bcfgs) == len(jcmds), "not %d jobs" % c.njob
jobs = []
for i in range(c.njob):
prefix = "%s.%d" % (c.job_prefix, i+1)
jcfg = {
'queue': c.pbs_queues[i],
'ppn': c.pbs_ppns[i],
'walltime': c.pbs_walltimes[i],
'mem': c.pbs_mems[i],
'email': c.pbs_email,
}
job = PbsJob.from_cfg(jcfg = jcfg, jcmds = jcmds[i], bcfgs = bcfgs[i],
prefix = prefix, njob = len(bcfgs[i]),
bash = c.bash, parallel = c.parallel)
jobs.append(job)
t = Table.read(c.ilist, format = 'ascii.tab')
nrow = len(t)
gts = [t['genotype'][x] for x in range(nrow) if t['type'][x] == 'Inbred']
gts = set(gts)
logging.debug("creating pseudo-refs for %d genomes" % len(gts))
print(" ".join(gts))
for gt in gts:
diro = "%s/%s" % (c.outdirs[0], gt)
mkdir(diro)
jobs[0].subjobs[0].add_cmd("%s consensus -f %s %s -s %s \
-c %s/25.chain -o %s/11_genome.fas" % \
(c.bcftools, c.genome, c.vcf, gt, diro, diro))
#jobs[1].subjobs[0].add_cmd("genome fasta %s" % diro)
#jobs[0].subjobs[0].add_cmd("genome blat %s" % diro)
#jobs[0].subjobs[0].add_cmd("genome bwa %s" % diro)
jobs[1].subjobs[0].add_cmd("genome bowtie %s" % diro)
jobs[2].subjobs[0].add_cmd("genome hisat %s" % diro)
for job in jobs:
job.write()
fj = "%s.sh" % c.job_prefix
create_job_chain([job.fname for job in jobs], fj)
logging.debug("job chain with %s jobs was created: %s" % (c.njob, fj))
def make_genomedir(species):
dirw = species
if species.isalnum():
dirw = op.join("/home/springer/zhoux379/data/genome", species)
logging.debug("converting species to directory: %s" % dirw)
if not op.isdir(dirw):
logging.debug("creating diretory: %s" % dirw)
mkdir(dirw)
return dirw
def make_genomedir(species):
dirw = species
if species.isalnum():
dirw = op.join("/home/springer/zhoux379/data/genome", species)
logging.debug("converting species to directory: %s" % dirw)
if not op.isdir(dirw):
logging.debug("creating diretory: %s" % dirw)
mkdir(dirw)
return dirw
def build_bwa(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/bwa")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.bwt") and not args.overwrite:
logging.debug("db.bwt already exists - skipped")
else:
sh("bwa index -a bwtsw -p %s/db %s" % (dirw, fg))
def build_bwa(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/bwa")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.bwt") and not args.overwrite:
logging.debug("db.bwt already exists - skipped")
else:
sh("bwa index -a bwtsw -p %s/db %s" % (dirw, fg))
def build_blat(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/blat")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if not args.overwrite and op.isfile('db.2bit'):
logging.debug("db.2bit already exists - skipped")
else:
sh("faToTwoBit %s db.2bit" % fg)
sh("blat db.2bit tmp.fas tmp.out -makeOoc=db.2bit.tile11.ooc")
if op.isfile("tmp.out"): os.remove("tmp.out")
def build_blat(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/blat")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if not args.overwrite and op.isfile('db.2bit'):
logging.debug("db.2bit already exists - skipped")
else:
sh("faToTwoBit %s db.2bit" % fg)
sh("blat db.2bit tmp.fas tmp.out -makeOoc=db.2bit.tile11.ooc")
if op.isfile("tmp.out"): os.remove("tmp.out")
def fasta(args):
"""
%prog fasta fastqfiles
Convert fastq to fasta and qual file.
"""
p = OptionParser(fasta.__doc__)
sp1.add_argument("--seqtk",
default=False,
action="store_true",
help="Use seqtk to convert")
p.set_outdir()
p.set_outfile(outfile=None)
opts, args = p.parse_args(args)
if len(args) < 1:
sys.exit(not p.print_help())
fastqfiles = args
outdir = args.outdir
if outdir and outdir != ".":
mkdir(outdir)
fastqfile = fastqfiles[0]
pf = op.basename(fastqfile)
gzinput = pf.endswith(".gz")
if gzinput:
pf = pf.rsplit(".", 1)[0]
pf, sf = pf.rsplit(".", 1)
if sf not in ("fq", "fastq"):
logging.debug("Assumed FASTA: suffix not `fq` or `fastq`")
return fastqfile, None
fastafile, qualfile = pf + ".fasta", pf + ".qual"
outfile = args.outfile or fastafile
outfile = op.join(outdir, outfile)
if args.seqtk:
if need_update(fastqfiles, outfile):
for i, fastqfile in enumerate(fastqfiles):
cmd = "seqtk seq -A {0} -L 30 -l 70".format(fastqfile)
# First one creates file, following ones append to it
sh(cmd, outfile=outfile, append=i)
else:
logging.debug("Outfile `{0}` already exists.".format(outfile))
return outfile, None
for fastqfile in fastqfiles:
SeqIO.convert(fastqfile, "fastq", fastafile, "fasta")
SeqIO.convert(fastqfile, "fastq", qualfile, "qual")
return fastafile, qualfile
def build_bowtie(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/bowtie2")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.rev.1.bt2") and not args.overwrite:
logging.debug("db.*.bt2 already exists - skipped")
else:
sh("rm -rf *")
sh("ln -sf %s db.fa" % fg)
# need to "module load bowtie2"
sh("bowtie2-build db.fa db")
def build_bowtie(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/bowtie2")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.rev.1.bt2") and not args.overwrite:
logging.debug("db.*.bt2 already exists - skipped")
else:
sh("rm -rf *")
sh("ln -sf %s db.fa" % fg)
# need to "module load bowtie2"
sh("bowtie2-build db.fa db")
def build_gatk(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/gatk")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.dict") and not args.overwrite:
logging.debug("db.dict already exists - skipped")
else:
if op.exists("db.fasta"): sh("rm db.fasta")
if op.exists("db.dict"): sh("rm db.dict")
sh("cp ../../10_genome.fna db.fasta")
sh("gatk CreateSequenceDictionary -R db.fasta")
sh("samtools faidx db.fasta")
def build_gatk(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/gatk")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
if op.isfile("db.dict") and not args.overwrite:
logging.debug("db.dict already exists - skipped")
else:
if op.exists("db.fasta"): sh("rm db.fasta")
if op.exists("db.dict"): sh("rm db.dict")
sh("cp ../../10_genome.fna db.fasta")
sh("gatk CreateSequenceDictionary -R db.fasta")
sh("samtools faidx db.fasta")
def build_hisat(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/hisat2")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
f_gtf = "../../50_annotation/10.gtf"
if op.isfile("db.1.ht2") and not args.overwrite:
logging.debug("db.1.ht2 already exists - skipped")
elif not op.isfile(f_gtf):
logging.error("no gtf file: f_gtf")
sys.exit()
else:
sh("hisat2_extract_exons.py %s > db.exon" % f_gtf)
sh("hisat2_extract_splice_sites.py %s > db.ss" % f_gtf)
sh("hisat2-build -p %d --ss db.ss --exon db.exon %s db" % (args.p, fg))
def build_star(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/star")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
f_gtf = "../../50_annotation/10.gtf"
if op.isfile("SA") and not args.overwrite:
logging.debug("SA already exists - skipped")
elif not op.isfile(f_gtf):
logging.error("no gtf file: %s" % f_gtf)
sys.exit()
else:
sh("STAR --runThreadN %d --runMode genomeGenerate --genomeDir %s \
--genomeFastaFiles %s --sjdbGTFfile %s" %
(args.p, ".", fg, f_gtf))
def build_star(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/star")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
f_gtf = "../../50_annotation/10.gtf"
if op.isfile("SA") and not args.overwrite:
logging.debug("SA already exists - skipped")
elif not op.isfile(f_gtf):
logging.error("no gtf file: %s" % f_gtf )
sys.exit()
else:
sh("STAR --runThreadN %d --runMode genomeGenerate --genomeDir %s \
--genomeFastaFiles %s --sjdbGTFfile %s" %
(args.p, ".", fg, f_gtf))
def build_hisat(args):
dirg, fg = get_genomedir(args.species)
dirw = op.join(dirg, "21_dbs/hisat2")
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
f_gtf = "../../50_annotation/10.gtf"
if op.isfile("db.1.ht2") and not args.overwrite:
logging.debug("db.1.ht2 already exists - skipped")
elif not op.isfile(f_gtf):
logging.error("no gtf file: f_gtf")
sys.exit()
else:
sh("hisat2_extract_exons.py %s > db.exon" % f_gtf)
sh("hisat2_extract_splice_sites.py %s > db.ss" % f_gtf)
sh("hisat2-build -p %d --ss db.ss --exon db.exon %s db" % (args.p, fg))
def __init__(self, filename, outputdir=None, format="fasta", mode="cycle"):
self.filename = filename
self.outputdir = outputdir
self.mode = mode
format = format or self._guess_format(filename)
logging.debug("format is %s" % format)
if format in ("fasta", "fastq"):
self.klass = "seqio"
elif format == "clust":
self.klass = "clust"
else:
self.klass = "txt"
self.format = format
mkdir(outputdir)
def merge_dirs(args):
diris, diro = args.diri, args.diro
mkdir(diro, overwrite=True)
for diri in args.diri:
for fn in os.listdir(diri):
fi = op.join(diri, fn)
fo = op.join(diro, fn)
if not op.isfile(fi): continue
if op.isfile(fo):
if not cmp(fi, fo):
if args.replace:
copy(fi, fo)
else:
print("%s/%s diff from %s/%s - skipped" %
(diri, fn, diro, fn))
else:
copy(fi, fo)
def merge(args):
"""
%prog merge merged_bams bams1_dir bams2_dir ...
Merge BAM files. Treat the bams with the same prefix as a set.
Output the commands first.
"""
from maize.apps.grid import MakeManager
p = OptionParser(merge.__doc__)
p.set_sep(sep="_", help="Separator to group per prefix")
opts, args = p.parse_args(args)
if len(args) < 2:
sys.exit(not p.print_help())
merged_bams = args[0]
bamdirs = args[1:]
mkdir(merged_bams)
bams = []
for x in bamdirs:
bams += glob(op.join(x, "*.bam"))
bams = [x for x in bams if "nsorted" not in x]
logging.debug("Found a total of {0} BAM files.".format(len(bams)))
sep = args.sep
key = lambda x: op.basename(x).split(sep)[0]
bams.sort(key=key)
mm = MakeManager()
for prefix, files in groupby(bams, key=key):
files = sorted(list(files))
nfiles = len(files)
source = " ".join(files)
target = op.join(merged_bams, op.basename(files[0]))
if nfiles == 1:
source = get_abs_path(source)
cmd = "ln -s {0} {1}".format(source, target)
mm.add("", target, cmd)
else:
cmd = "samtools merge -@ 8 {0} {1}".format(target, source)
mm.add(files, target, cmd, remove=True)
mm.write()
def merge(args):
"""
%prog merge merged_bams bams1_dir bams2_dir ...
Merge BAM files. Treat the bams with the same prefix as a set.
Output the commands first.
"""
from maize.apps.grid import MakeManager
p = OptionParser(merge.__doc__)
p.set_sep(sep="_", help="Separator to group per prefix")
opts, args = p.parse_args(args)
if len(args) < 2:
sys.exit(not p.print_help())
merged_bams = args[0]
bamdirs = args[1:]
mkdir(merged_bams)
bams = []
for x in bamdirs:
bams += glob(op.join(x, "*.bam"))
bams = [x for x in bams if "nsorted" not in x]
logging.debug("Found a total of {0} BAM files.".format(len(bams)))
sep = args.sep
key = lambda x: op.basename(x).split(sep)[0]
bams.sort(key=key)
mm = MakeManager()
for prefix, files in groupby(bams, key=key):
files = sorted(list(files))
nfiles = len(files)
source = " ".join(files)
target = op.join(merged_bams, op.basename(files[0]))
if nfiles == 1:
source = get_abs_path(source)
cmd = "ln -s {0} {1}".format(source, target)
mm.add("", target, cmd)
else:
cmd = "samtools merge -@ 8 {0} {1}".format(target, source)
mm.add(files, target, cmd, remove=True)
mm.write()
def prepare(cfg):
cfg = cfg['prepare']
dirw, qry, tgt = cfg['dirw'], cfg['qry'], cfg['tgt']
qry_fas, tgt_fas = cfg['qry_fas'], cfg['tgt_fas']
tmpdir = cfg['temp_dir']
npieces = int(cfg['npieces'])
if not op.isdir(dirw):
logging.debug("making directory: %s" % dirw)
mkdir(dirw)
os.chdir(dirw)
subdirs = ['01_tgt_genome', '02_qry_genome']
for subdir in subdirs:
if not op.isdir(subdir):
logging.debug("making directory: %s" % subdir)
mkdir(subdir)
if op.isfile(tgt_fas):
fo = "raw.fa"
if tgt_fas.endswith(".fa.gz") or tgt_fas.endswith(".fas.gz"):
fo = "raw.fa.gz"
sh("ln -sf %s 01_tgt_genome/%s" % (tgt_fas, fo))
else:
logging.error("%s not exist" % qry_fas)
if op.isfile(qry_fas):
fo = "raw.fa"
if qry_fas.endswith(".fa.gz") or qry_fas.endswith(".fas.gz"):
fo = "raw.fa.gz"
sh("ln -sf %s 02_qry_genome/%s" % (qry_fas, fo))
else:
logging.error("%s not exist" % qry_fas)
sh("genome fasta %s/%s" % (dirw, "01_tgt_genome"))
sh("genome blat %s/%s" % (dirw, "01_tgt_genome"))
sh("genome fasta %s/%s" % (dirw, "02_qry_genome"))
sh("genome blat %s/%s" % (dirw, "02_qry_genome"))
sh("bed filter --minsize 1000 02_qry_genome/16.gap.bed > 04.qry.gap.bed")
sh("subtractBed -nonamecheck -a 02_qry_genome/15.bed -b 04.qry.gap.bed | bed filter -min 100 - | bed makewindow -w 100000 -s 95000 - > 05.qry.clean.bed")
sh("bed size 05.qry.clean.bed")
sh("fasta extract 02_qry_genome/11_genome.fas 05.qry.clean.bed > 06.qry.fas")
sh("fasta split --N %d %s %s" % (npieces, '06.qry.fas', diro1))
def tsvs(args):
"""
%prog tsvs excelfile
Convert all worksheets in EXCEL to tsv files.
"""
excelfile = args.excel
odir = args.outdir
sep = args.sep
xl = pd.ExcelFile(excelfile)
sheets = xl.sheet_names
print("will convert %d sheets under %s" % (len(sheets), odir))
mkdir(odir)
suf = '.tsv' if sep == '\t' else '.csv'
for sheet in sheets:
fo = "%s/%s%s" % (odir, sheet, suf)
print(" writing %s" % fo)
df = pd.read_excel(excelfile, sheet_name=sheet, header=0)
df.to_csv(fo, sep=sep, header=True, index=False)
def check_cfg_mapping(c):
c.outdirs = c.outdir.split(",")
assert len(c.outdirs) == 2, "not 2 outdirs: %s" % c.outdir
for subdir in [c.dirw, c.temp_dir] + c.outdirs:
if not op.isdir(subdir):
mkdir(subdir)
for fn in [c.ilist, c.genome, c.gff]:
assert op.isfile(fn), "cannot read %s" % fn
for key in 'samtools parallel sambamba bcftools bedtools'.split():
fp = which(c[key])
assert fp is not None, "not executable: %s" % c[key]
c[key] = fp
c.paired = str2bool(c.paired)
if c.mapper == 'bwa':
c.bwa = which(c.bwa)
assert c.bwa is not None, "not executable: %s" % c.bwa
elif c.mapper == 'hisat2':
c.hisat2 = which(c.hisat2)
assert c.hisat2 is not None, "not executable: %s" % c.hisat2
elif c.mapper == 'bowtie2':
c.bowtie2 = which(c.bowtie2)
assert c.bowtie2 is not None, "not executable: %s" % c.bowtie2
else:
logging.error("unsupported mapper: %s" % c.mapper)
sys.exit(1)
njob = 3
c.pbs_walltimes = c.pbs_walltime.split(",")
c.pbs_ppns = c.pbs_ppn.split(",")
c.pbs_queues = c.pbs_queue.split(",")
assert njob == len(c.pbs_queues) == len(c.pbs_walltimes) == len(
c.pbs_ppns), "not %d jobs: %s" % (njob, c.pbs_queue)
c.njob = njob
return c
def check_cfg_mapping(c):
c.outdirs = c.outdir.split(",")
assert len(c.outdirs) == 2, "not 2 outdirs: %s" % c.outdir
for subdir in [c.dirw, c.temp_dir] + c.outdirs:
if not op.isdir(subdir):
mkdir(subdir)
for fn in [c.ilist, c.genome, c.gff]:
assert op.isfile(fn), "cannot read %s" % fn
for key in 'samtools parallel sambamba bcftools bedtools'.split():
fp = which(c[key])
assert fp is not None, "not executable: %s" % c[key]
c[key] = fp
c.paired = str2bool(c.paired)
if c.mapper == 'bwa':
c.bwa = which(c.bwa)
assert c.bwa is not None, "not executable: %s" % c.bwa
elif c.mapper == 'hisat2':
c.hisat2 = which(c.hisat2)
assert c.hisat2 is not None, "not executable: %s" % c.hisat2
elif c.mapper == 'bowtie2':
c.bowtie2 = which(c.bowtie2)
assert c.bowtie2 is not None, "not executable: %s" % c.bowtie2
else:
logging.error("unsupported mapper: %s" % c.mapper)
sys.exit(1)
njob = 3
c.pbs_walltimes = c.pbs_walltime.split(",")
c.pbs_ppns = c.pbs_ppn.split(",")
c.pbs_queues = c.pbs_queue.split(",")
assert njob == len(c.pbs_queues) == len(c.pbs_walltimes) == len(c.pbs_ppns), "not %d jobs: %s" % (njob, c.pbs_queue)
c.njob = njob
return c
def split_old(args):
fi, dirw = op.realpath(args.fi), op.realpath(args.outdir)
n = args.N
if not op.exists(dirw):
mkdir(dirw)
else:
sh("rm -rf %s/*" % dirw)
cdir = os.path.dirname(os.path.realpath(__file__))
cwd = os.getcwd()
os.chdir(dirw)
sh("ln -sf %s part.fas" % fi)
sh("pyfasta split -n %d part.fas" % n)
sh("rm part.fas part.fas.*")
digit = ndigit(n)
sizes = []
for i in range(0,n):
fmt = "part.%%0%dd.fas" % digit
fp = fmt % i
sizes.append(os.stat(fp).st_size)
sizes.sort()
print("size range: %s - %s" % (prettysize(sizes[0]), prettysize(sizes[n-1])))
def check_cfg_index(c, noutdir = 1, njob = 3):
c.outdirs = c.outdir.split(",")
assert len(c.outdirs) == noutdir, "not %s outdirs: %s" % (noutdir, c.outdir)
for subdir in [c.dirw, c.temp_dir] + c.outdirs:
if not op.isdir(subdir):
mkdir(subdir)
for fn in [c.ilist, c.vcf, c.genome, c.gff]:
assert op.isfile(fn), "cannot read %s" % fn
for key in 'bcftools'.split():
fp = which(c[key])
assert fp is not None, "not executable: %s" % c[key]
c[key] = fp
c.pbs_walltimes = c.pbs_walltime.split(",")
c.pbs_ppns = c.pbs_ppn.split(",")
c.pbs_queues = c.pbs_queue.split(",")
c.pbs_mems = c.pbs_mem.split(",")
assert njob == len(c.pbs_queues) == len(c.pbs_walltimes) == len(c.pbs_ppns) == len(c.pbs_mems), "not %d jobs: %s" % (njob, c.pbs_queue)
c.njob = njob
return c
description='rename fastq files')
parser.add_argument('diri', help='FROM directory path')
parser.add_argument('diro', help='TO directory path')
parser.add_argument('list', help='read list (sid Readfile1[ Readfile2])')
args = parser.parse_args()
#dirw = '/home/springer/zhoux379/projects/3rnaseq/cache'
#diri = '/scratch.global/tenders/3RNA_0418/fastq_gz_files'
#diro = '10.fastq'
diri, diro, ilist = args.diri, args.diro, args.list
if not op.isfile(ilist):
logging.error("cannot read %s" % ilist)
sys.exit()
if not op.isdir(diro):
mkdir(diro)
t = Table.read(ilist, format='ascii.tab')
paired = False
if 'Readfile2' in t.colnames:
paired = True
tag = "paired" if paired else "single"
logging.debug("proceed as %s-end reads" % tag)
for i in range(len(t)):
if paired:
sid, fq1, fq2 = t['sid'][i], t['Readfile1'][i], t['Readfile2'][i]
fq1, fq2 = op.join(diri, fq1), op.join(diri, fq2)
assert op.isfile(fq1), "cannot access fq1: %s" % fq1
assert op.isfile(fq2), "cannot access fq2: %s" % fq2
cmd = "cp %s %s/%s_1.fq.gz" % (fq1, diro, sid)
def run_blat(cfg):
cfg = cfg['blat']
dirw, jobpre, diro1, diro2 = \
cfg['dirw'], cfg['job_prefix'], cfg['outdir1'], cfg['outdir2']
qry, tgt = cfg['qry'], cfg['tgt']
parallel = cfg['parallel']
npieces, npieces2 = int(cfg['npieces']), int(cfg['npieces2'])
pbs_template, pbs_queue, pbs_walltime, pbs_ppn, pbs_email = \
cfg['pbs_template'], cfg['pbs_queue'], cfg['pbs_walltime'], \
cfg['pbs_ppn'], cfg['pbs_email']
if not op.isdir(dirw): mkdir(dirw)
os.chdir(dirw)
for subdir in [diro1, diro2]:
if not op.isdir(subdir):
mkdir(subdir)
dirt = "01_tgt_genome"
dirq = "02_qry_genome"
tgt_fas = "%s/11_genome.fas" % dirt
qry_fas = "%s/11_genome.fas" % dirq
tgt_2bit = "%s/21.blat/db.2bit" % dirt
qry_2bit = "%s/21.blat/db.2bit" % dirq
tgt_ooc = "%s/21.blat/db.2bit.tile11.ooc" % dirt
tgt_size = "%s/15.sizes" % dirt
qry_size = "%s/15.sizes" % dirq
tgt_size_bed = "%s/15.bed" % dirt
qry_size_bed = "%s/15.bed" % dirq
tgt_gap = "%s/16.gap.bed" % dirt
qry_gap = "%s/16.gap.bed" % dirq
for fn in [tgt_fas, qry_fas, tgt_2bit, qry_2bit, tgt_ooc,
tgt_size, qry_size, tgt_size_bed, qry_size_bed, tgt_gap, qry_gap]:
if not op.isfile(fn):
logging.error("%s is not there" % fn)
sys.exit()
pbs_queues = pbs_queue.split(",")
pbs_ppns = pbs_ppn.split(",")
pbs_walltimes = pbs_walltime.split(",")
njob = len(pbs_queues)
assert len(pbs_walltimes) == njob, "not %d jobs" % njob
assert len(pbs_ppns) == njob, "not %d jobs" % njob
fbs = ["%sb.%d.sh" % (jobpre, i+1) for i in range(njob)]
fjs = ["%sj.%d.pbs" % (jobpre, i+1) for i in range(njob)]
bcmds, jcmds = [], []
#1 blat
cmds = []
bcmds.append(cmds)
prepre = "part.%%0%dd" % ndigit(npieces-1)
jcmds.append([
"let i=${PBS_ARRAYID}",
"cd %s" % dirw,
"printf -v pre %s \"$i\"" % prepre,
"pblat %s %s/${pre}.fas -threads=%s -ooc=%s %s/${pre}.psl" % \
(tgt_2bit, diro1, pbs_ppns[0], tgt_ooc, diro2)
])
#2 process blat
bcmds.append([
"pslCat -nohead 12.blat/part.*.psl > 12.psl",
"psl qcoord 12.psl %s > 13.psl" % qry_size,
"pslCheck -querySizes=%s -targetSizes=%s -pass=14.check.psl 13.psl" %
(qry_size, tgt_size),
"axtChain -linearGap=medium -psl 14.check.psl %s %s 21.chain" %
(tgt_2bit, qry_2bit),
"chainPreNet 21.chain %s %s 23.chain" % (tgt_size, qry_size),
"chain 2bed --qry 23.chain | sortBed -i stdin | \
mergeBed -i stdin > 23.bed",
"subtractBed -a %s -b %s -nonamecheck | \
subtractBed -a stdin -b 23.bed -nonamecheck | \
bed filter --minsize 50 - > 24.nov.bed" %
(qry_size_bed, qry_gap),
"seqret.pl -d %s -b 24.nov.bed -o 24.nov.fas" % qry_fas,
"rm 23.chain 23.bed",
"fasta split --N %d %s %s" % (npieces2, '24.nov.fas', '25.nov'),
])
jcmds.append([
"cd %s" % dirw,
"bash %s" % fbs[1],
])
#3 blat nov
cmds = []
bcmds.append(cmds)
prepre = "part.%%0%dd" % ndigit(npieces2-1)
jcmds.append([
"let i=${PBS_ARRAYID}",
"cd %s" % dirw,
"printf -v pre %s \"$i\"" % prepre,
"pblat %s %s/${pre}.fas -threads=%s -ooc=%s %s/${pre}.psl" % \
(tgt_2bit, '25.nov', pbs_ppns[2], tgt_ooc, '25.nov')
])
#4 process blat
bcmds.append([
"pslCat -nohead 25.nov/part.*.psl > 25.nov.psl",
"psl qcoord 25.nov.psl %s > 26.psl" % qry_size,
"pslCheck -querySizes=%s -targetSizes=%s -pass=27.check.psl 26.psl" %
(qry_size, tgt_size),
"pslCat 14.check.psl 27.check.psl > 31.1.psl",
"pslSwap 31.1.psl 41.1.psl",
"rm 25.nov.psl 26.psl",
"axtChain -linearGap=medium -psl 31.1.psl %s %s 31.2.chain" % (tgt_2bit, qry_2bit),
"chainPreNet 31.2.chain %s %s 31.3.chain" % (tgt_size, qry_size),
"chainSwap 31.3.chain 31.3.q.chain",
"chainNet 31.3.chain %s %s 31.5.net 31.5.q.net" % (tgt_size, qry_size),
"netChainSubset 31.5.net 31.3.chain stdout | chainSort stdin 31.5.chain",
"netChainSubset 31.5.q.net 31.3.q.chain stdout | chainSort stdin 31.5.q.chain",
"chainNet 31.5.q.chain %s %s /dev/null 31.8.net" % (qry_size, tgt_size),
"netChainSubset 31.8.net 31.3.chain 31.8.chain",
"axtChain -linearGap=medium -psl 41.1.psl %s %s 41.2.chain" % (qry_2bit, tgt_2bit),
"chainPreNet 41.2.chain %s %s 41.3.chain" % (qry_size, tgt_size),
"chainSwap 41.3.chain 41.3.q.chain",
"chainNet 41.3.chain %s %s 41.5.net 41.5.q.net" % (qry_size, tgt_size),
"netChainSubset 41.5.net 41.3.chain stdout | chainSort stdin 41.5.chain",
"netChainSubset 41.5.q.net 41.3.q.chain stdout | chainSort stdin 41.5.q.chain",
"chainNet 41.5.q.chain %s %s /dev/null 41.8.net" % (tgt_size, qry_size),
"netChainSubset 41.8.net 41.3.chain 41.8.chain",
])
jcmds.append([
"cd %s" % dirw,
"bash %s" % fbs[3],
])
#5 process vnt
bcmds.append([
"snp2vcf.pl -i snp -o snp.vcf -s %s" % qry,
])
jcmds.append([
"cd %s/31.9" % dirw,
"bash %s" % fbs[4],
])
assert len(bcmds) == njob, "not %d jobs" % njob
assert len(jcmds) == njob, "not %d jobs" % njob
for i in range(njob):
fb, fj = fbs[i], fjs[i]
if op.isfile(fb):
os.remove(fb)
if len(bcmds[i]) > 0:
fhb = open(fb, "w")
fhb.write("\n".join(bcmds[i]) + "\n")
fhb.close()
pbsjob = PbsJob(queue = pbs_queues[i],
ppn = pbs_ppns[i],
walltime = pbs_walltimes[i],
email = pbs_email,
cmds = "\n".join(jcmds[i])
)
pbsjob.write(fj)
logging.debug("%s job scripts were created" % njob)
eprint("qsub -t 0-%d %s" % (npieces-1, fjs[0]))
eprint("qsub -W depend=afterok: %s" % fjs[1])
eprint("qsub -t 0-%d %s" % (npieces2-1, fjs[2]))
eprint("qsub -W depend=afterok: %s" % fjs[3])
eprint("qsub -W depend=afterok: %s" % fjs[4])
def check_cfg_mapping(c, noutdir = 4, njob = 2):
c.outdirs = c.outdir.split(",")
assert len(c.outdirs) == noutdir, "not %s outdirs: %s" % (noutdir, c.outdir)
for subdir in [c.dirw, c.temp_dir] + c.outdirs:
if not op.isdir(subdir):
mkdir(subdir)
for fn in [c.ilist, c.vcf, c.gene_bed]:
assert op.isfile(fn), "cannot read %s" % fn
for key in 'samtools parallel sambamba htseq bcftools bedtools'.split():
fp = which(c[key])
assert fp is not None, "not executable: %s" % c[key]
c[key] = fp
c.paired = str2bool(c.paired)
assert c.stranded in ['yes', 'no', 'reverse'], "unknown stranded option: %s" % c.stranded
if c.mapper == 'tophat2':
c.tophat2 = which(c.tophat2)
assert c.tophat2 is not None, "not executable: %s" % c.tophat2
elif c.mapper == 'hisat2':
c.hisat2= which(c.hisat2)
assert c.hisat2 is not None, "not executable: %s" % c.hisat2
elif c.mapper == 'star':
c.star= which(c.star)
assert c.star is not None, "not executable: %s" % c.star
else:
logging.error("unsupported mapper: %s" % c.mapper)
sys.exit(1)
assert op.isdir(c.genomedir), "cannot access %s" % c.genomedir
t = Table.read(c.ilist, format = 'ascii.tab')
if 'genome' not in t[0]:
genomeb = 'B73c'
logging.debug("no 'genome' column detected: use %s" % genomeb)
t.add_column(Column([genomeb] * len(t)), name = 'genome')
c.t = t
genomes = set()
for i in range(len(t)):
gts = t['genome'][i].split(",")
for gt in gts:
genomes.add(gt)
genomes = sorted(list(genomes))
logging.debug("checking %d genomes" % len(genomes))
c.genomes = dict()
for gt in genomes:
c.genomes[gt] = dict()
dirg = "%s/%s" % (c.genomedir, gt)
dbpre = ''
if c.mapper == 'tophat2':
dbpre = "%s/21.bowtie2/db" % dirg
assert op.isfile("%s.4.bt2" % dbpre), "no %s db-index: %s" % (c.mapper, dbpre)
elif c.mapper == 'hisat2':
dbpre = "%s/21.hisat2/db" % dirg
assert op.isfile("%s.8.ht2" % dbpre), "no %s db-index: %s" % (c.mapper, dbpre)
elif c.mapper == 'star':
dbpre = "%s/21.star" % dirg
assert op.isfile("%s/SA" % dbpre), "no %s db-index: %s" % (c.mapper, dbpre)
c.genomes[gt]['db'] = dbpre
gff = "%s/51.gff" % dirg
assert op.isfile(gff), "no gff for %s: %s" % (gff, gt)
c.genomes[gt]['gff'] = gff
c.pbs_walltimes = c.pbs_walltime.split(",")
c.pbs_ppns = c.pbs_ppn.split(",")
c.pbs_queues = c.pbs_queue.split(",")
assert njob == len(c.pbs_queues) == len(c.pbs_walltimes) == len(c.pbs_ppns), "not %d jobs: %s" % (njob, c.pbs_queue)
c.njob = njob
return c
)
parser.add_argument('diri', help = 'FROM directory path')
parser.add_argument('diro', help = 'TO directory path')
parser.add_argument('list', help = 'read list (sid Readfile1[ Readfile2])')
args = parser.parse_args()
#dirw = '/home/springer/zhoux379/projects/3rnaseq/cache'
#diri = '/scratch.global/tenders/3RNA_0418/fastq_gz_files'
#diro = '10.fastq'
diri, diro, ilist = args.diri, args.diro, args.list
if not op.isfile(ilist):
logging.error("cannot read %s" % ilist)
sys.exit()
if not op.isdir(diro):
mkdir(diro)
t = Table.read(ilist, format = 'ascii.tab')
paired = False
if 'Readfile2' in t.colnames:
paired = True
tag = "paired" if paired else "single"
logging.debug("proceed as %s-end reads" % tag)
for i in range(len(t)):
if paired:
sid, fq1, fq2 = t['sid'][i], t['Readfile1'][i], t['Readfile2'][i]
fq1, fq2 = op.join(diri, fq1), op.join(diri, fq2)
assert op.isfile(fq1), "cannot access fq1: %s" % fq1
assert op.isfile(fq2), "cannot access fq2: %s" % fq2
cmd = "cp %s %s/%s_1.fq.gz" % (fq1, diro, sid)
def entrez(args):
"""
%prog entrez <filename|term>
`filename` contains a list of terms to search. Or just one term. If the
results are small in size, e.g. "--format=acc", use "--batchsize=100" to speed
the download.
"""
p = OptionParser(entrez.__doc__)
allowed_databases = {"fasta": ["genome", "nuccore", "nucgss", "protein", "nucest"],
"asn.1": ["genome", "nuccore", "nucgss", "protein", "gene"],
"xml" : ["genome", "nuccore", "nucgss", "nucest", "gene"],
"gb" : ["genome", "nuccore", "nucgss"],
"est" : ["nucest"],
"gss" : ["nucgss"],
"acc" : ["nuccore"],
}
valid_formats = tuple(allowed_databases.keys())
valid_databases = ("genome", "nuccore", "nucest", "nucgss", "protein", "gene")
sp1.add_argument("--noversion", dest="noversion",
default=False, action="store_true",
help="Remove trailing accession versions")
sp1.add_argument("--format", default="fasta", choices=valid_formats,
help="download format [default: %default]")
sp1.add_argument("--database", default="nuccore", choices=valid_databases,
help="search database [default: %default]")
sp1.add_argument("--retmax", default=1000000, type="int",
help="how many results to return [default: %default]")
sp1.add_argument("--skipcheck", default=False, action="store_true",
help="turn off prompt to check file existence [default: %default]")
sp1.add_argument("--batchsize", default=500, type="int",
help="download the results in batch for speed-up [default: %default]")
p.set_outdir(outdir=None)
sp1.add_argument("--outprefix", default="out",
help="output file name prefix [default: %default]")
p.set_email()
opts, args = p.parse_args(args)
if len(args) != 1:
sys.exit(p.print_help())
filename, = args
if op.exists(filename):
pf = filename.rsplit(".", 1)[0]
list_of_terms = [row.strip() for row in open(filename)]
if args.noversion:
list_of_terms = [x.rsplit(".", 1)[0] for x in list_of_terms]
else:
pf = filename
# the filename is the search term
list_of_terms = [filename.strip()]
fmt = args.format
database = args.database
batchsize = args.batchsize
assert database in allowed_databases[fmt], \
"For output format '{0}', allowed databases are: {1}".\
format(fmt, allowed_databases[fmt])
assert batchsize >= 1, "batchsize must >= 1"
if " " in pf:
pf = args.outprefix
outfile = "{0}.{1}".format(pf, fmt)
outdir = args.outdir
if outdir:
mkdir(outdir)
# If noprompt, will not check file existence
if not outdir:
fw = must_open(outfile, "w", checkexists=True, \
skipcheck=args.skipcheck)
if fw is None:
return
seen = set()
totalsize = 0
for id, size, term, handle in batch_entrez(list_of_terms, retmax=args.retmax, \
rettype=fmt, db=database, batchsize=batchsize, \
email=args.email):
if outdir:
outfile = urljoin(outdir, "{0}.{1}".format(term, fmt))
fw = must_open(outfile, "w", checkexists=True, \
skipcheck=args.skipcheck)
if fw is None:
continue
rec = handle.read()
if id in seen:
logging.error("Duplicate key ({0}) found".format(rec))
continue
totalsize += size
print >> fw, rec
print >> fw
seen.add(id)
if seen:
print >> sys.stderr, "A total of {0} {1} records downloaded.".\
format(totalsize, fmt.upper())
return outfile