def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest',
parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
class MergerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='mergerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_merger(self, name):
try:
return config.create_merger(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an merger of type %s !
This could indicate a bug in create_merger_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def testOpal(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('opal')
def testMuscle(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('muscle')
def _impl_test_merger(self, name):
filename = data_source_path('merger1.fasta')
alignment1 = Alignment()
alignment1.read_filepath(filename, 'FASTA')
filename = data_source_path('merger2.fasta')
alignment2 = Alignment()
alignment2.read_filepath(filename, 'FASTA')
aln = self.get_merger('%s merger' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment1,
alignment2,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('merger_result.fasta')
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
self.assertEquals(reference_aln, a)
class MergerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='mergerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_merger(self, name):
try:
return config.create_merger(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an merger of type %s !
This could indicate a bug in create_merger_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def testOpal(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('opal')
def testMuscle(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('muscle')
def _impl_test_merger(self, name):
filename = data_source_path('merger1.fasta')
alignment1 = Alignment()
alignment1.read_filepath(filename, 'FASTA')
filename = data_source_path('merger2.fasta')
alignment2 = Alignment()
alignment2.read_filepath(filename, 'FASTA')
aln = self.get_merger('%s merger' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment1,
alignment2,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('merger_result.fasta')
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
self.assertEquals(reference_aln, a)
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
class TreeEstimatorTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_tree_estimator(self, name):
try:
return config.create_tree_estimator(name=name, temp_fs=self.ts)
except RuntimeError:
log_exception(_LOG)
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_tree_estimator_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_tree_estimator(self, name, datatype, partitions):
filename = data_source_path('anolis.fasta')
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
te = self.get_tree_estimator(name)
if te is None:
_LOG.warn("test%s skipped" % name)
return
alignment.datatype = datatype
a = te.run(alignment=alignment,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
def testRaxml(self):
filename = data_source_path('mafft.anolis.fasta')
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_tree_estimator('raxml', datatype="DNA", partitions=[("DNA", 1, 1456)])
class SateTestCase(unittest.TestCase):
def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
def tear_down(self):
self.register_files()
self.remove_dir()
def _main_execution(self, args, stdout=None, stderr=None, rc=0):
try:
cmd = "import sys; from sate.mainsate import sate_main; sate_main(%s)[0] or sys.exit(1)" % repr(args)
invoc = [sys.executable, '-c', cmd]
_LOG.debug("Command:\n\tpython -c " + repr(cmd))
p = subprocess.Popen(invoc,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
(o, e) = p.communicate()
r = p.wait()
if r != rc:
_LOG.error("exit code (%s) did not match %s" % (r,
rc))
_LOG.error("here is the stdout:\n%s" % o)
_LOG.error("here is the stderr:\n%s" % e)
self.assertEquals(r, rc)
if stderr is not None:
self.assertEquals(e, stderr)
if stdout is not None:
self.assertEquals(o, stdout)
except Exception, v:
#self.assertEquals(str(v), 5)
raise
def __init__(self, config):
"""Uses a configuration object `cfg` to get reference for the tools the user
has chosen.
"""
try:
max_mem_mb = config.sate.max_mem_mb
self._temp_fs = TempFS()
self.aligner = config.create_aligner(temp_fs=self._temp_fs)
self.aligner.max_mem_mb = max_mem_mb
self.merger = config.create_merger(temp_fs=self._temp_fs)
self.merger.max_mem_mb = max_mem_mb
self.tree_estimator = config.create_tree_estimator(
temp_fs=self._temp_fs)
self.raxml_tree_estimator = config.create_tree_estimator(
name='Raxml', temp_fs=self._temp_fs)
except AttributeError:
raise
raise ValueError(
"config cannot be None unless all of the tools are passed in.")
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting', parent=os.curdir)
class AlignerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_aligner(self, name):
try:
return config.create_aligner(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_aligner_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_aligner(self, name, fn):
filename = data_source_path(fn)
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
aln = self.get_aligner('%s' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('%s.%s' % (name, fn))
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, reference_fn))
if reference_aln != a:
i = 1
while True:
nrfn = reference_fn + '.' + str(i)
if os.path.exists(nrfn):
reference_aln = Alignment()
reference_aln.read_filepath(nrfn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, nrfn))
if reference_aln == a:
self.assertEquals(reference_aln, a)
return True
i += 1
else:
self.assertEquals(reference_aln, a)
def testClustalW2(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('clustalw2', 'anolis.fasta')
def testOpal(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('opal', 'anolis.fasta')
def testMafft(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('mafft', 'anolis.fasta')
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
def setUp(self):
self.ts = TempFS()
class TreeEstimatorTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest',
parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
try:
self.ts.remove_dir(dir)
except ValueError:
pass
def get_tree_estimator(self, name):
try:
return config.create_tree_estimator(name=name, temp_fs=self.ts)
except RuntimeError:
log_exception(_LOG)
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_tree_estimator_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_tree_estimator(self, name, datatype, partitions, **kwargs):
num_cpus = kwargs.get('num_cpus', None)
filename = data_source_path('anolis.fasta')
md = MultiLocusDataset()
md.read_files(seq_filename_list=[filename], datatype=datatype)
md.relabel_for_sate()
# alignment = Alignment()
# alignment.read_filepath(filename, 'FASTA')
te = self.get_tree_estimator(name)
if te is None:
_LOG.warn("test%s skipped" % name)
return
# alignment.datatype = datatype
if num_cpus:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True,
num_cpus=num_cpus)
else:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
def testRaxml(self):
# if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_tree_estimator('raxml',
datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTree(self):
config.fasttree.add_option(
'options',
StringUserSetting(name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree',
datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTreeMP(self):
config.fasttree.add_option(
'options',
StringUserSetting(name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree',
datatype="DNA",
partitions=[("DNA", 1, 1456)],
num_cpus=2)
class AlignerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_aligner(self, name):
try:
return config.create_aligner(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_aligner_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_aligner(self, name, fn):
filename = data_source_path(fn)
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
aln = self.get_aligner('%s' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('%s.%s' % (name, fn))
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, reference_fn))
if reference_aln != a:
i = 1
while True:
nrfn = reference_fn + '.' + str(i)
if os.path.exists(nrfn):
reference_aln = Alignment()
reference_aln.read_filepath(nrfn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, nrfn))
if reference_aln == a:
self.assertEquals(reference_aln, a)
return True
i += 1
else:
self.assertEquals(reference_aln, a)
def testClustalW2(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('clustalw2', 'anolis.fasta')
def testOpal(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('opal', 'anolis.fasta')
def testMafft(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('mafft', 'anolis.fasta')
class TreeEstimatorTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
try:
self.ts.remove_dir(dir)
except ValueError:
pass
def get_tree_estimator(self, name):
try:
return config.create_tree_estimator(name=name, temp_fs=self.ts)
except RuntimeError:
log_exception(_LOG)
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_tree_estimator_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_tree_estimator(self, name, datatype, partitions, **kwargs):
num_cpus = kwargs.get('num_cpus', None)
filename = data_source_path('anolis.fasta')
md = MultiLocusDataset()
md.read_files(seq_filename_list=[filename],
datatype=datatype)
md.relabel_for_sate()
# alignment = Alignment()
# alignment.read_filepath(filename, 'FASTA')
te = self.get_tree_estimator(name)
if te is None:
_LOG.warn("test%s skipped" % name)
return
# alignment.datatype = datatype
if num_cpus:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True,
num_cpus=num_cpus)
else:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
def testRaxml(self):
# if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_tree_estimator('raxml', datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTree(self):
config.fasttree.add_option('options', StringUserSetting(
name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree', datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTreeMP(self):
config.fasttree.add_option('options', StringUserSetting(
name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree', datatype="DNA",
partitions=[("DNA", 1, 1456)], num_cpus=2)
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix="alignerTesting", parent=os.curdir)
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='mergerTesting', parent=os.curdir)
class TempFSTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def testCreate(self):
self.assertTrue(True)
def testNoTopLevel(self):
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
cur_parent = os.path.abspath(os.curdir)
self.assertRaises(ValueError, self.ts.create_temp_subdir, prefix='bogus', parent=cur_parent)
self.assertEquals(self.ts.get_remaining_directories(), set())
def testBadParForTop(self):
fn = 'THANKS.txt'
if os.path.exists(fn) and (not os.path.isdir(fn)):
self.assertRaises(OSError, self.ts.create_top_level_temp, prefix='bogus', parent=fn)
else:
_LOG.warn("test of create_top_level_temp with file as parent skipped because '%s' does not exist" % fn)
bogus_par = 'bogus_par'
if os.path.exists(bogus_par):
_LOG.warn("test of create_top_level_temp with non existent parent skipped because '%s' exists" % bogus_par)
else:
self.assertRaises(OSError, self.ts.create_top_level_temp, prefix='bogus', parent=bogus_par)
def testTop(self):
d = self.ts.create_top_level_temp(prefix='bogus 4 testing', parent=os.curdir)
self.assertEquals(os.path.realpath(d), d)
self.assertEquals(self.ts.top_level_temp, d)
self.assertEquals(os.path.abspath(d), d)
self.assertTrue(os.path.exists(d))
self.assertTrue(os.path.isdir(d))
self.assertTrue(d in self.ts.get_remaining_directories())
# there can be only on top
self.assertRaises(AssertionError, self.ts.create_top_level_temp, prefix='bogus_for_testing', parent=os.curdir)
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError, self.ts.create_subdir, 'bogus')
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='bogus', parent=os.curdir)
# but be created inside
ssd = os.path.join(d, 'bogussd')
sd = self.ts.create_subdir(ssd)
self.assertEquals(sd, ssd)
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(sd))
self.assertTrue(os.path.isdir(sd))
nssd = os.path.join(ssd, 'nested')
nsd = self.ts.create_subdir(nssd)
self.assertEquals(nsd, nssd)
self.assertTrue(nsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(nsd))
self.assertTrue(os.path.isdir(nsd))
tsd = self.ts.create_temp_subdir(prefix='bogus', parent=d)
self.assertEquals(os.path.realpath(tsd), tsd)
self.assertEquals(os.path.abspath(tsd), tsd)
self.assertTrue(os.path.exists(tsd))
self.assertTrue(os.path.isdir(tsd))
self.assertTrue(tsd in self.ts.get_remaining_directories())
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertEquals(len(self.ts.get_remaining_directories()), 4)
# create tempdir in nested
tnsd = self.ts.create_temp_subdir(prefix='tempinnested', parent=nsd)
self.assertEquals(os.path.realpath(tnsd), tnsd)
self.assertEquals(os.path.abspath(tnsd), tnsd)
self.assertTrue(os.path.exists(tnsd))
self.assertTrue(os.path.isdir(tnsd))
self.assertTrue(tnsd in self.ts.get_remaining_directories())
# subdirectories within create_temp_subdir should work...
innermost = os.path.join(tnsd, 'innermost')
innermostsd = self.ts.create_subdir(innermost)
self.assertEquals(innermostsd, innermost)
self.assertTrue(innermostsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(innermostsd))
self.assertTrue(os.path.isdir(innermostsd))
self.assertRaises(ValueError, self.ts.remove_dir, 'THANKS.txt')
self.assertEquals(self.ts.remove_dir(sd), True)
self.assertFalse(os.path.exists(sd))
self.assertFalse(os.path.exists(innermostsd))
# removing sd will remove nsd (because it is inside sd), so
# trying to created directories in that location should fail
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='tempinnested', parent=nsd)
self.assertTrue(not os.path.exists(sd))
self.assertTrue(os.path.exists(tsd))
self.assertEquals(self.ts.remove_dir(d), True)
self.assertFalse(os.path.exists(tsd))
self.assertRaises(ValueError, self.ts.remove_dir, tsd)
self.assertRaises(ValueError, self.ts.remove_dir, d)
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='bogus', parent=d)