def runtest():
failed = False
sraob = sra.SRA('ERR726985', directory='./pyrpipe_sratest')
if not sraob.fastq_exists():
pu.print_boldred('Test failed')
failed = True
pu.print_notification('Cleaning up...')
sraob.delete_fastq()
os.rmdir(sraob.directory)
if failed:
pu.print_boldred('Paired end test failed')
failed = False
sraob = sra.SRA('SRR2134545', directory='./pyrpipe_sratest')
if not sraob.fastq_exists():
pu.print_boldred('Test failed')
failed = True
pu.print_notification('Cleaning up...')
sraob.delete_fastq()
os.rmdir(sraob.directory)
if failed:
pu.print_boldred('Single end test failed')
failed = False
if not failed:
pu.print_green(
'\n#####################All Tests Passed#####################\n')
os.rmdir('./pyrpipe_sratest')
def test_sra():
srrID = testVars.srr
print("Testing SRA...")
testDir = testVars.testDir
newOb = sra.SRA(srrID, testDir)
assert newOb.srr_accession == srrID, "Failed SRA init"
assert newOb.download_sra() == True, "Failed to download SRA file"
assert newOb.sraFileExistsLocally(
) == True, "Failed to locate .sra file on disk"
##test fasterq-dump
assert newOb.run_fasterqdump(delete_sra=True, **{
"-f": "",
"-t": testDir
}) == True, "Failed FQ dump"
assert newOb.fastqFilesExistsLocally(
) == True, "Failed to locate .fastq files on disk"
assert newOb.sraFileExistsLocally(
) != True, "Failed to delete .sra files from disk"
st = pyrpipe_session.save_session(filename="mySession",
add_timestamp=False,
out_dir=testVars.testDir)
assert st == True, "Session save failed"
st = pyrpipe_session.restore_session(testVars.testDir +
"/mySession.pyrpipe")
assert st == True, "Session restore failed"
assert newOb.fastqFilesExistsLocally(
) == True, "Failed to locate .fastq files on disk after restore"
def test_sra2():
srrID=testVars.srr
testDir=testVars.testDir
newOb=sra.SRA(srrID,testDir+"/testsra")
assert newOb.srr_accession==srrID,"Failed SRA init"
assert newOb.fastq_exists()==True, "Failed to locate .fastq files on disk"
assert newOb.sra_exists()==False, "SRA file exists. "
#delete downloaded files
assert newOb.delete_fastq()==True, "Failed to delete .fastq files from disk"
def test_sra():
srrID=testVars.srr
print ("Testing SRA...")
testDir=testVars.testDir
newOb=sra.SRA(srrID,testDir+"/testsra")
assert newOb.srr_accession==srrID,"Failed SRA init"
assert newOb.fastq_exists()==True, "Failed to locate .fastq files on disk"
assert newOb.sra_exists()==False, "Deleted SRA file exists. "
assert newOb.download_sra()==True, "Failed to download SRA file"
assert newOb.sra_exists()==True, "Failed to locate .sra file on disk"
assert newOb.delete_sra()==True, "Failed to delete .sra files from disk"
assert newOb.sra_exists()==False, "Deleted SRA file exists. "
#delete downloaded files
assert newOb.delete_fastq()==True, "Failed to delete .fastq files from disk"
def test_sra():
srrID = testVars.srr
print("Testing SRA...")
testDir = testVars.testDir
newOb = sra.SRA(srrID, testDir + "/testsra")
assert newOb.srr_accession == srrID, "Failed SRA init"
assert newOb.download_sra() == True, "Failed to download SRA file"
assert newOb.sraFileExistsLocally(
) == True, "Failed to locate .sra file on disk"
##test fasterq-dump
assert newOb.run_fasterqdump(delete_sra=True, **{
"-f": "",
"-t": testDir
}) == True, "Failed FQ dump"
assert newOb.fastqFilesExistsLocally(
) == True, "Failed to locate .fastq files on disk"
assert newOb.sraFileExistsLocally(
) != True, "Failed to delete .sra files from disk"
#delete downloaded files
assert newOb.delete_fastq(
) == True, "Failed to delete .fastq files from disk"
def test_sra():
srrID = testVars.srr
print("Testing SRA...")
testDir = testVars.testDir
newOb = sra.SRA(srrID, testDir)
assert newOb.srr_accession == srrID, "Failed SRA init"
assert newOb.fastq_exists(
) == True, "Failed to locate .fastq files on disk"
assert newOb.sra_exists() == False, "Failed to delete .sra files from disk"
st = pyrpipe_session.save_session(filename="mySession",
add_timestamp=False,
out_dir=testVars.testDir)
assert st == True, "Session save failed"
st = pyrpipe_session.restore_session(testVars.testDir +
"/mySession.pyrpipe")
assert st == True, "Session restore failed"
assert newOb.fastq_exists(
) == True, "Failed to locate .fastq files on disk after restore"
assert newOb.delete_fastq() == True, "failed to delete fastq"
tr=assembly.Trinity()
#trOpts={"--seqType":"fq","--left":"/home/usingh/work/urmi/hoap/test/test_Trinity_Assembly/reads.left.fq.gz",
# "--right":"/home/usingh/work/urmi/hoap/test/test_Trinity_Assembly/reads.right.fq.gz","--output":"/home/usingh/work/urmi/hoap/test/trinity_testout","--max_memory":"2G"}
#tr.run_trinity(verbose=True,**trOpts)
tr.perform_assembly(sraOb,verbose=True)
"""
"""
sr=sra.SRA(scan_path="/home/usingh/work/urmi/hoap/test/SRR5507495/")
# scan a dir to crteate SRA objects
#subdirs=next(os.walk(testDir))[1]
subdirs=[os.path.join(testDir, o) for o in os.listdir(testDir)
if os.path.isdir(os.path.join(testDir,o))]
sraList=[]
for d in subdirs:
try:
thisSRA=sra.SRA(scan_path=d)
sraList.append(thisSRA)
print("FOUND data in "+d)
except:
print("no data in "+d)
"""
print("fstqTEST")
sob = sra.SRA('SRR5507495', testDir)
sob.download_fastq(procs=8)
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 10 19:06:34 2019
@author: usingh
"""
from pyrpipe import sra, mapping, assembly, qc, tools, pyrpipe_utils
workingDir = "/home/usingh/work/urmi/hoap/test"
athalRunsSmol = ['SRR1583780', 'SRR5507495', 'SRR5507442']
sraObjects = []
for x in athalRunsSmol:
thisSraOb = sra.SRA(x, workingDir)
if thisSraOb.downloadSRAFile():
sraObjects.append(thisSraOb)
else:
print("Download failed:" + x)
print("Following runs downloaded:")
for ob in sraObjects:
print(ob.srrAccession)
for ob in sraObjects:
ob.runFasterQDump(deleteSRA=True,
**{
"-e": "8",
"-f": "",
"-t": workingDir
def test_pipeline1():
sraOb = sra.SRA(srr, workingDir)
st = sraOb.download_sra()
assert st == True, "SRA download failed"
st = sraOb.run_fasterqdump(delete_sra=False,
**{
"-e": "8",
"-f": "",
"-t": workingDir
})
assert st == True, "fqdump failed"
bbdOpts = {
"ktrim": "r",
"k": "23",
"mink": "11",
"qtrim": "'rl'",
"trimq": "10",
"--": ("-Xmx2g", ),
"ref": testVars.bbdukAdapters
}
bbdOb = qc.BBmap(**bbdOpts)
st = sraOb.perform_qc(bbdOb)
assert st == True, "bbduk failed"
tgOpts = {
"--cores": "10",
"-o": testVars.testDir,
"--paired": "",
"--": (fq1, fq2)
}
tg = qc.Trimgalore(**tgOpts)
st = sraOb.perform_qc(tg)
assert st == True, "tg failed"
#runbowtie2
bt = mapping.Bowtie2(bowtie2_index="")
assert bt.check_index() == False, "Failed bowtie2 check_index"
st = bt.build_index(testVars.testDir + "/btIndex", "bowtieIndex",
testVars.genome)
assert st == True, "Failed to build bowtie2 index"
st = bt.perform_alignment(sraOb)
assert os.path.isfile(st) == True, "bowtie failed"
hsOpts = {"--dta-cufflinks": "", "-p": "8"}
hs = mapping.Hisat2(hisat2_index="", **hsOpts)
st = hs.build_index(testVars.testDir, "hisatindex", testVars.genome)
assert st == True, "Failed to build hisat2 index"
#perform alignment with sraobject
st = hs.perform_alignment(sraOb)
assert os.path.isfile(st) == True, "hisat failed"
hisatSam = st
samOb = tools.Samtools(**{"-@": "8"})
bam = samOb.sam_sorted_bam(hisatSam, delete_sam=False, delete_bam=False)
assert os.path.isfile(bam) == True, "sam to bam failed"
stie = assembly.Stringtie(reference_gtf=testVars.gtf)
result = stie.perform_assembly(bam, out_dir=testVars.testDir)
assert pu.check_files_exist(result) == True, "Failed stringtie"
tr = assembly.Trinity()
tr_out = tr.perform_assembly(sraOb, verbose=True)
assert pu.check_files_exist(tr_out) == True, "Failed stringtie"
kl = quant.Kallisto(kallisto_index="")
assert kl.check_index() == False, "Failed kallisto check_index"
st = kl.build_index(index_path=testVars.testDir + "/kallistoIndex",
index_name="kalIndex",
fasta=testVars.cdna)
assert st == True, "Failed to build kallisto index"
st = kl.perform_quant(sraOb)
assert os.path.isdir(st) == True, "Failed to run kallisto"
sl = quant.Salmon(salmon_index="")
assert sl.check_index() == False, "Failed salmon check_index"
st = sl.build_index(index_path=testVars.testDir + "/salmonIndex",
index_name="salIndex",
fasta=testVars.cdna)
assert st == True, "Failed to build salmon index"
st = sl.perform_quant(sraOb)
assert os.path.isdir(st) == True, "Failed to run salmon"
#new tests
# test samtools
sam=testDir+"/test_files/athaliana/mapping/hisat2.sam"
sm=tools.Samtools(threads=5)
bam1=sm.sam_to_bam(sam,out_suffix="test2",threads=3, delete_sam=False,verbose=True,quiet=False,logs=True,objectid="NA",**{"-@":"4"})
print(bam1)
bam2=sm.sort_bam(bam1,out_suffix="test2",threads=2,delete_bam=False,verbose=True,quiet=False,logs=True,objectid="NA")
print(bam2)
#bam=sm.sam_sorted_bam(sam,delete_sam=False,delete_bam=False)
txd=tools.Transdecoder()
infa="/Users/usingh/work/urmi/tests/txd/test.fa"
outdir=txd.run_transdecoder_longorfs(infa,out_dir="/Users/usingh/work/urmi/tests/txd/mtout1")
print(outdir)
poutdir="/Users/usingh/work/urmi/tests/txd/mypredout"
predout=txd.run_transdecoder_predict(infa,longorfs_dir=outdir,out_dir=poutdir)
print(predout)
newSRA=sra.SRA('SRR5507343',testDir)
newSRA.download_fastq()
#run trimgalore
tg=qc.Trimgalore(threads=4) #specify to use 8 cores
bd=qc.BBmap(threads=4,max_memory=1)
newSRA.perform_qc(bd)
#newSRA.perform_qc(tg)
genome=testVars.genome)
hsOpts = {"--dta-cufflinks": "", "-p": "8"}
hs = mapping.Hisat2(index=testVars.testDir + "/hisatindex",
genome=testVars.genome,
**hsOpts)
star = mapping.Star(index=os.path.join(testVars.testDir, "starIndex"),
genome=testVars.genome)
samOb = tools.Samtools()
stie = assembly.Stringtie()
kl = quant.Kallisto(index=testVars.testDir + "/kallistoIndex/kalIndex",
transcriptome=testVars.cdna)
sl = quant.Salmon(index=testVars.testDir + "/salmonIndex/salIndex",
transcriptome=testVars.cdna_big)
#sra ob
sraOb = sra.SRA(srr, workingDir)
st = sraOb.fastq_exists()
assert st == True, "fasterq-dump failed"
def test_pipeline1():
st = sraOb.trim(bbdOb).align(hs).assemble(stie).quant(kl)
assert st != None, "pipeline 1 failed"
def test_pipeline2():
st = sraOb.trim(tg).align(hs).assemble(stie).quant(kl)
assert st != None, "pipeline 2 failed"
def test_pipeline3():
progresslog.write(error + '\n')
continue
#convert bam to fastq
st = biobb.bamtofq(bam, sampleid)
if not st:
print('bam to fastq failed.')
#log this message
error = sampleid + '\tbam to fastq failed'
progresslog.write(error + '\n')
continue
#create SRA object
fq1 = os.path.join(bam_dir, sampleid + '_1.fastq')
fq2 = os.path.join(bam_dir, sampleid + '_2.fastq')
sobj = sra.SRA(fastq=fq1, fastq2=fq2)
#check files exist
if not _dryrun and not sobj.fastq_exists():
print('Error creating SRA obj')
error = sampleid + '\tFastq not found'
progresslog.write(error + '\n')
continue
if runalign:
#perform alignment
#start star alignment
star_bam = starobj.perform_alignment(sobj) #returns bam path
#delete fastq
sobj.delete_fastq()
if not star_bam:
print('Star failed for ' + bam)
#check if hisat is sucessful
if not hisatSam:
raise Exception("ERROR: Hisat failed")
#remove qc corrected fastq
sraOb.deleteFastqFiles()
#run sam to sorted bam then run stringtie
gtfS=stieOb.runStringtie(samtOb.samToSortedBam(hisatSam,10,deleteSam=True,deleteOriginalBam=True),deleteInputBam=True,proc=10)
"""
btIndex="/home/usingh/work/urmi/hoap/test/bowtieIndex/rRNAindex"
#riboseq SRR3590744
sraOb=sra.SRA(srr_accession='SRR5507495',location=testDir)
#download sra
sraOb.download_sra()
#run fastqdump;delete sra when done
sraOb.run_fasterqdump(delete_sra=False,**{"-f":"","-t":testDir})
tgOb=qc.Trimgalore()
sraOb.perform_qc(tgOb)
#pathToAdapters="/home/usingh/lib_urmi/softwares/bbmap/resources/adapters2.fa"
#bbdOpts={"ktrim":"r","k":"23","mink":"11","qtrim":"'rl'","trimq":"10","--":("-Xmx2g",),"ref":pathToAdapters}
#bbdOb=qc.BBmap(**bbdOpts)
#sraOb.perform_qc(bbdOb)
#status=bbdOb.performCleaning(sraOb,"/home/usingh/work/urmi/hoap/test/bowtieIndex/euk_combined_rRNA.fa")
