def assertEqualFolders(self, reffolder, resfolder, html=None, outputfolder=None):
if TestAugustus.opt_compare:
if html is None:
html = self.opt_html
if outputfolder is None:
diff = comp.compare_folder(reffolder,
resfolder,
html=html)
else:
diff = comp.compare_folder(reffolder,
resfolder,
html=html,
outputfolder=outputfolder)
self.assertEqual(diff, '', diff)
def test_alternatives_from_sampling(self):
os.chdir(default_wd)
resfolder = resultdir + self.test_alternatives_from_sampling.__name__ + '/'
os.mkdir(resfolder)
with open(resfolder + 'augustus_tmp.gff', 'w') as file:
cmd = [
augustusbin, '../../examples/autoAug/genome.fa',
'--species=caenorhabditis', '--alternatives-from-sampling=on',
'--minexonintronprob=0.08', '--minmeanexonintronprob=0.4',
'--maxtracks=3'
]
p = subprocess.Popen(cmd,
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output file
self.assertTrue(os.path.isfile(resfolder + 'augustus_tmp.gff'),
'Output file was not created as expected!')
afilter.pred(resfolder + 'augustus_tmp.gff',
resfolder + 'augustus.gff')
os.remove(resfolder + 'augustus_tmp.gff')
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(
refdir + self.test_alternatives_from_sampling.__name__ + '/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def test_format_and_error_out(self):
os.chdir(default_wd)
resfolder = resultdir + self.test_format_and_error_out.__name__ + '/'
os.mkdir(resfolder)
cmd = [
augustusbin, '../../examples/autoAug/genome.fa',
'--species=caenorhabditis', '--gff3=on', '--softmasking=1',
'--outfile=' + resfolder + 'augustus_tmp.gff3',
'--errfile=' + resfolder + 'augustus.err'
]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
p.stdout.close()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output file
self.assertTrue(os.path.isfile(resfolder + 'augustus_tmp.gff3'),
'Output file was not created as expected!')
afilter.pred(resfolder + 'augustus_tmp.gff3',
resfolder + 'augustus.gff3')
os.remove(resfolder + 'augustus_tmp.gff3')
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(
refdir + self.test_format_and_error_out.__name__ + '/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def test_utr_on(self):
os.chdir(default_wd)
resfolder = resultdir + self.test_utr_on.__name__ + '/'
os.mkdir(resfolder)
with open(resfolder + 'aug_utr_on_tmp.gff', 'w') as file:
p = subprocess.Popen([
augustusbin, '--species=human', '--UTR=on', '--softmasking=0',
'../../examples/example.fa'
],
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True)
rc = p.wait()
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
self.assertEqual(rc, 0, 'Returncode not 0!')
self.assertTrue(os.path.isfile((resfolder + 'aug_utr_on_tmp.gff')),
'Output file was not created as expected!')
# filter output file
afilter.pred(resfolder + 'aug_utr_on_tmp.gff',
resfolder + 'aug_utr_on.gff')
os.remove(resfolder + 'aug_utr_on_tmp.gff')
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(refdir + self.test_utr_on.__name__ +
'/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def test_iterative_prediction_with_hints(self):
os.chdir(default_wd)
resfolder = resultdir + self.test_iterative_prediction_with_hints.__name__ + '/'
proc_list = []
if not os.path.isfile('data/tmp/chr2L.sm.fa'):
init_test_data()
os.mkdir(resfolder)
for i in range(0, 3):
with open(resfolder + 'aug.nasonia.hints.' + str(i) + '_tmp.gff',
'w') as file:
proc_list.append(
subprocess.Popen([
augustusbin, '--species=nasonia',
'data/tmp/chr2L.sm.fa', '--softmasking=on',
'--predictionStart=' + str(i * 2000000),
'--predictionEnd=' + str((i + 1) * 2000000 + 50000),
'--hintsfile=../docs/tutorial2015/results/hints.gff',
'--extrinsicCfgFile=extrinsic.M.RM.E.W.cfg'
],
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True))
for p in proc_list:
p.wait()
for p in proc_list:
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output
for i in range(0, 3):
source = resfolder + 'aug.nasonia.hints.' + str(i) + '_tmp.gff'
self.assertTrue(os.path.isfile(source),
'Output file was not created as expected!')
target = resfolder + 'aug.nasonia.hints.' + str(i) + '.gff'
afilter.pred(source, target)
os.remove(source)
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(
refdir + self.test_iterative_prediction_with_hints.__name__ +
'/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def test_cgp(self):
os.chdir(default_wd)
os.chdir('../../examples/cgp')
resfolder = '../../tests/' + resultdir.replace(
'../', '') + self.test_cgp.__name__ + '/'
reffolder = '../../tests/' + refdir.replace(
'../', '') + self.test_cgp.__name__ + '/'
os.mkdir(resfolder)
with open(resfolder + 'output_tmp.txt', 'w') as file:
cmd = [
augustusbin,
'--species=human',
'--speciesfilenames=genomes.tbl',
'--treefile=tree.nwk',
'--alnfile=aln.maf',
'--softmasking=0',
'--alternatives-from-evidence=0', # removes warning
'--/CompPred/outdir=' + resfolder
]
p = subprocess.Popen(cmd,
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output files
for file in os.listdir(resfolder):
filename = os.fsdecode(file)
if filename.endswith('.gff'):
afilter.cgp(
resfolder + filename, resfolder + '/' +
filename.replace('.gff', '.filtered.gff'))
os.remove(resfolder + filename)
afilter.cgp_out(resfolder + 'output_tmp.txt', resfolder + 'output.txt')
os.remove(resfolder + 'output_tmp.txt')
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(reffolder,
resfolder,
html=TestAugustus.opt_html,
outputfolder=default_wd +
'/output_html/')
self.assertEqual(diff, '', diff)
def cgp_with_db_execution(self, resfolder, reffolder, *args):
os.mkdir(resfolder)
proc_list = []
# create groups according to the configured number of cpus
grouped_args = [iter(args)] * TestAugustus.cpuno
# parallel execution of the commands of each group
for arg_list in itertools.zip_longest(*grouped_args):
proc_list = []
for cmd, filename in filter(None, arg_list):
with open(filename, 'w') as file:
proc_list.append(
subprocess.Popen(cmd,
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True))
for p in proc_list:
p.wait()
for p in proc_list:
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output prediction files
for subdir, dirs, files in os.walk(resfolder):
for file in files:
filename = os.fsdecode(file)
if filename.endswith('.gff'):
afilter.cgp(
subdir + '/' + filename, subdir + '/' +
filename.replace('.gff', '.filtered.gff'))
os.remove(subdir + '/' + filename)
elif filename.endswith('.out'):
afilter.cgp_out(
subdir + '/' + filename,
subdir + '/' + filename.replace('_tmp', ''))
os.remove(subdir + '/' + filename)
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(reffolder,
resfolder,
html=TestAugustus.opt_html,
outputfolder=default_wd +
'/output_html/')
self.assertEqual(diff, '', diff)
def test_iterative_prediction(self):
os.chdir(default_wd)
resfolder = resultdir + self.test_iterative_prediction.__name__ + '/'
species_list = ['nasonia', 'zebrafish', 'tomato']
proc_list = []
# run augustus several times with different parameter sets
os.mkdir(resfolder)
for species in species_list:
with open(resfolder + 'aug.' + species + '.1-1M_tmp.gff',
'w') as file:
proc_list.append(
subprocess.Popen([
augustusbin, '--species=' + species,
'data/tmp/chr2L.sm.fa', '--softmasking=on',
'--predictionEnd=1000000'
],
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True))
for p in proc_list:
p.wait()
for p in proc_list:
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output
for species in species_list:
source = resfolder + 'aug.' + species + '.1-1M_tmp.gff'
self.assertTrue(os.path.isfile(source),
'Output file was not created as expected!')
target = resfolder + 'aug.' + species + '.1-1M.gff'
afilter.pred(source, target)
os.remove(source)
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(
refdir + self.test_iterative_prediction.__name__ + '/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def test_hints_MPE(self):
dirname = self.test_hints_MPE.__name__
fname = "aug_hints_MPE.gff"
tfname = "aug_hints_MPE_tmp.gff"
os.chdir(default_wd)
resfolder = resultdir + dirname + '/'
os.mkdir(resfolder)
with open(resfolder + tfname, 'w') as file:
p = subprocess.Popen([
augustusbin, '--species=human',
'--hintsfile=../../examples/hints.gff',
'--extrinsicCfgFile=../../config/extrinsic/extrinsic.MPE.cfg',
'../../examples/example.fa'
],
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True)
rc = p.wait()
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
self.assertEqual(rc, 0, 'Returncode not 0!')
self.assertTrue(os.path.isfile((resfolder + tfname)),
'Output file was not created as expected!')
# filter output file
afilter.pred(resfolder + tfname, resfolder + fname)
os.remove(resfolder + tfname)
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(refdir + dirname + '/',
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)
def training_new_species(self, crf):
speciesname = 'test_aug_dev_species'
# Remove test species folder.
# Just in case the deletion fails for whatever reason.
if os.path.exists('../../config/species/' + speciesname):
shutil.rmtree('../../config/species/' + speciesname)
resfolder = (
resultdir +
self.test_training_new_species_crf.__name__) if crf else (
resultdir + self.test_training_new_species.__name__) + '/'
reffolder = (
refdir + self.test_training_new_species_crf.__name__
) if crf else (refdir + self.test_training_new_species.__name__) + '/'
os.mkdir(resfolder)
# call script to initialzie new species
p = subprocess.Popen([
'perl', '../../scripts/new_species.pl', '--species=' + speciesname,
'--AUGUSTUS_CONFIG_PATH=../../config'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
stdout = p.stdout.read()
p.stdout.close()
p.stderr.close()
#print(stdout)
if error:
print(error)
# training
p = subprocess.Popen([
'../../bin/etraining',
'../../docs/tutorial2015/results/genes.gb.train',
'--species=' + speciesname
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
stdout = p.stdout.read()
p.stdout.close()
p.stderr.close()
#print(stdout)
self.assertEqual(error, '', error)
# test
with open(resfolder + 'test_tmp.out', 'w') as file:
cmd = [
augustusbin, '../../docs/tutorial2015/results/genes.gb.test',
'--species=' + speciesname, '--softmasking=0',
'--AUGUSTUS_CONFIG_PATH=../../config'
]
if (crf):
cmd.append('--CRF=on')
cmd.append('--CRF_N=2')
cmd.append('--UTR=off')
p = subprocess.Popen(cmd,
stdout=file,
stderr=subprocess.PIPE,
universal_newlines=True)
p.wait()
error = p.stderr.read()
p.stderr.close()
self.assertEqual(error, '', error)
# filter output file
self.assertTrue(os.path.isfile(resfolder + 'test_tmp.out'),
'Output file was not created as expected!')
afilter.eval(resfolder + 'test_tmp.out', resfolder + 'test.out')
os.remove(resfolder + 'test_tmp.out')
# move new species to result folder
shutil.move('../../config/species/' + speciesname, resfolder)
# compare results
if TestAugustus.opt_compare:
diff = comp.compare_folder(reffolder,
resfolder,
html=TestAugustus.opt_html)
self.assertEqual(diff, '', diff)