def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
class MergerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='mergerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_merger(self, name):
try:
return config.create_merger(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an merger of type %s !
This could indicate a bug in create_merger_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def testOpal(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('opal')
def testMuscle(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('muscle')
def _impl_test_merger(self, name):
filename = data_source_path('merger1.fasta')
alignment1 = Alignment()
alignment1.read_filepath(filename, 'FASTA')
filename = data_source_path('merger2.fasta')
alignment2 = Alignment()
alignment2.read_filepath(filename, 'FASTA')
aln = self.get_merger('%s merger' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment1,
alignment2,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('merger_result.fasta')
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
self.assertEquals(reference_aln, a)
class MergerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='mergerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_merger(self, name):
try:
return config.create_merger(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an merger of type %s !
This could indicate a bug in create_merger_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def testOpal(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('opal')
def testMuscle(self):
if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_merger('muscle')
def _impl_test_merger(self, name):
filename = data_source_path('merger1.fasta')
alignment1 = Alignment()
alignment1.read_filepath(filename, 'FASTA')
filename = data_source_path('merger2.fasta')
alignment2 = Alignment()
alignment2.read_filepath(filename, 'FASTA')
aln = self.get_merger('%s merger' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment1,
alignment2,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('merger_result.fasta')
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
self.assertEquals(reference_aln, a)
def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
def __init__(self, config):
"""Uses a configuration object `cfg` to get reference for the tools the user
has chosen.
"""
try:
max_mem_mb = config.sate.max_mem_mb
self._temp_fs = TempFS()
self.aligner = config.create_aligner(temp_fs=self._temp_fs)
self.aligner.max_mem_mb = max_mem_mb
self.hmmeralign = config.create_aligner(temp_fs=self._temp_fs,
name="hmmeralign")
self.merger = config.create_merger(temp_fs=self._temp_fs)
self.merger.max_mem_mb = max_mem_mb
self.tree_estimator = config.create_tree_estimator(
temp_fs=self._temp_fs)
self.raxml_tree_estimator = config.create_tree_estimator(
name='Raxml', temp_fs=self._temp_fs)
self.subsets = {} # needed for pastamerger
self.alignmentjobs = [] # needed for pastamerger
self.treeshrink_wrapper = config.create_treeshrink_wrapper(
temp_fs=self._temp_fs)
except AttributeError:
raise
raise ValueError(
"config cannot be None unless all of the tools are passed in.")
class PastaTeam (object):
'''A blob for holding the appropriate merger, alignment, and tree_estimator tools
as well as the TempFS object that keeps track of the directories that have
been created by this process.
'''
def __init__(self, config):
"""Uses a configuration object `cfg` to get reference for the tools the user
has chosen.
"""
try:
max_mem_mb = config.sate.max_mem_mb
self._temp_fs = TempFS()
self.aligner = config.create_aligner(temp_fs=self._temp_fs)
self.aligner.max_mem_mb = max_mem_mb
self.hmmeralign = config.create_aligner(temp_fs=self._temp_fs, name = "hmmeralign")
self.merger = config.create_merger(temp_fs=self._temp_fs)
self.merger.max_mem_mb = max_mem_mb
self.tree_estimator = config.create_tree_estimator(temp_fs=self._temp_fs)
self.raxml_tree_estimator = config.create_tree_estimator(name='Raxml', temp_fs=self._temp_fs)
# self.subsetalignmentjobs=[] #needed for interuptable job
self.subsets = {} # needed for pastamerger
self.alignmentjobs = [] # needed for pastamerger
self.interruptible=False
self.config = config
except AttributeError:
raise
raise ValueError("config cannot be None unless all of the tools are passed in.")
def get_temp_fs(self):
return self._temp_fs
temp_fs = property(get_temp_fs)
def change_aligner_to_custom(self):
self.aligner=self.config.create_aligner(temp_fs=self._temp_fs,custom=True)
def make_picklable(self):
self._temp_fs.make_picklable()
self.hmmeralign.make_picklable()
def make_unpickled(self):
self._temp_fs.make_unpickled()
self.hmmeralign.make_unpickled()
class SateTestCase(unittest.TestCase):
def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
def tear_down(self):
self.register_files()
self.remove_dir()
def _main_execution(self, args, stdout=None, stderr=None, rc=0):
try:
cmd = "import sys; from pasta.mainpasta import pasta_main; pasta_main(%s)[0] or sys.exit(1)" % repr(args)
invoc = [sys.executable, '-c', cmd]
_LOG.debug("Command:\n\tpython -c " + repr(cmd))
p = subprocess.Popen(invoc,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
(o, e) = p.communicate()
r = p.wait()
if r != rc:
_LOG.error("exit code (%s) did not match %s" % (r,
rc))
_LOG.error("here is the stdout:\n%s" % o)
_LOG.error("here is the stderr:\n%s" % e)
self.assertEquals(r, rc)
if stderr is not None:
self.assertEquals(e, stderr)
if stdout is not None:
self.assertEquals(o, stdout)
except Exception, v:
#self.assertEquals(str(v), 5)
raise
def setUp(self):
self.ts = TempFS()
class SateTestCase(unittest.TestCase):
def set_up(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='runSateTest',
parent=TESTS_DIR)
self.job_name = 'satejob' + self.random_id(8)
self.dirs = set([self.ts.top_level_temp])
self.paths = set()
def tear_down(self):
self.register_files()
self.remove_dir()
def _main_execution(self, args, stdout=None, stderr=None, rc=0):
try:
cmd = "import sys; from pasta.mainpasta import pasta_main; pasta_main(%s)[0] or sys.exit(1)" % repr(args)
invoc = [sys.executable, '-c', cmd]
_LOG.debug("Command:\n\tpython -c " + repr(cmd))
p = subprocess.Popen(invoc,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
(o, e) = p.communicate()
r = p.wait()
if r != rc:
_LOG.error("exit code (%s) did not match %s" % (r,
rc))
_LOG.error("here is the stdout:\n%s" % o)
_LOG.error("here is the stderr:\n%s" % e)
self.assertEqual(r, rc)
if stderr is not None:
self.assertEqual(e, stderr)
if stdout is not None:
self.assertEqual(o, stdout)
except Exception as v:
#self.assertEquals(str(v), 5)
raise
def _exe_run_sate(self, args, stdout=None, stderr=None, rc=0):
script_path = os.path.join(pasta.pasta_home_dir(), 'run_sate.py')
if isinstance(args, str):
arg_list = args.split()
else:
arg_list = args
cmd = ['python', script_path] + arg_list
_LOG.debug("Command:\n\t" + " ".join(cmd))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
o, e = p.communicate()
exit_code = p.wait()
if exit_code != rc:
_LOG.error("exit code (%s) did not match %s" % (exit_code,
rc))
_LOG.error("here is the stdout:\n%s" % o)
_LOG.error("here is the stderr:\n%s" % e)
self.assertEqual(exit_code, rc)
if stdout != None:
self.assertEqual(o, stdout)
if stderr != None:
self.assertEqual(e, stderr)
def _exe(self, args):
return pasta_main(args)
def parse_fasta_file(self, file):
if isinstance(file, str):
_LOG.info('parsing fasta file {0!r}...'.format(file))
file_stream = open(file, 'rU')
else:
file_stream = file
line_iter = iter(file_stream)
data = {}
seq = StringIO()
name = None
for i, line in enumerate(line_iter):
l = line.strip()
if l.startswith('>'):
if name:
data[name] = seq.getvalue().upper()
name = l[1:]
seq = StringIO()
else:
seq.write(l.replace(' ', ''))
if name:
data[name] = seq.getvalue().upper()
file_stream.close()
return data
def parse_score_file(self, file_obj):
file_stream = file_obj
if isinstance(file_obj, str):
_LOG.info('parsing score file {0!r}...'.format(file_obj))
file_stream = open(file_obj, 'rU')
return(float(file_stream.read().strip()))
def parse_score_arg(self, arg):
if isinstance(arg, float):
return arg
return self.parse_score_file(arg)
def parse_tree_arg(self, arg):
if isinstance(arg, dendropy.Tree):
return arg
return self.parse_tree_file(arg)
def parse_tree_file(self, file_obj):
file_stream = file_obj
if isinstance(file_obj, str):
_LOG.info('parsing tree file {0!r}...'.format(file_obj))
file_stream = open(file_obj, 'rU')
t = dendropy.Tree()
t.read_from_stream(file_stream, schema='newick')
file_stream.close()
return t
def assertSameTrees(self, tree_list, percent_tol=1e-6):
if len(tree_list) < 2:
return
tree1 = self.parse_tree_arg(tree_list.pop(0))
for t in tree_list:
tree2 = self.parse_tree_arg(t)
self.assertEqualTreePair(tree1, tree2, percent_tol)
def assertEqualTreePair(self, tree1, tree2, percent_tol=1e-6):
self.assertEqual(sorted(tree1.taxon_set.labels()),
sorted(tree2.taxon_set.labels()))
nodes1 = [n for n in tree1.postorder_node_iter()]
nodes2 = [n for n in tree2.postorder_node_iter()]
self.assertEqual(len(nodes1), len(nodes2))
for i, n1 in enumerate(nodes1):
n2 = nodes2[i]
if n1.taxon is not None:
self.assertTrue(n2.taxon is not None)
self.assertEqual(n1.taxon.label, n2.taxon.label)
else:
self.assertEqual(n2.taxon, None)
if n1.edge.length is not None:
self.assertTrue(n2.edge.length is not None)
self.assertApproxEqual(n1.edge.length, n2.edge.length,
percent_tol)
else:
self.assertEqual(n2.edge.length, None)
def assertSameScores(self, score_list, percent_tol=1e-6):
if len(score_list) < 2:
return
score1 = self.parse_score_arg(score_list.pop(0))
for s in score_list:
score2 = self.parse_score_arg(s)
self.assertApproxEqual(score1, score2, percent_tol=percent_tol)
def assertApproxEqual(self, x, y, percent_tol=1e-6):
self.assertTrue(
((abs(x-y) / ((abs(x)+abs(y))/2))*100) < percent_tol)
def parseSequenceArg(self, seq_arg):
if isinstance(seq_arg, dict):
return seq_arg
else:
return self.parse_fasta_file(seq_arg)
def remove_gaps(self, sequence_dict):
sd = self.parseSequenceArg(sequence_dict)
new_sd = {}
for name, seq in list(sd.items()):
new_seq = re.sub(r'[-?]', '', seq)
if new_seq != '':
new_sd[name] = new_seq
return new_sd
def concatenate_sequences(self, seq_data_list):
taxa = set()
data_sets = []
for f in seq_data_list:
seqs = self.parseSequenceArg(f)
taxa.update(list(seqs.keys()))
data_sets.append(seqs)
data = {}
for t in taxa:
data[t] = ''
for ds in data_sets:
for name in taxa:
data[name] += ds.get(name, '')
return data
def assertSameTaxa(self, seq_data_list):
if len(seq_data_list) < 2:
return
seqs1 = self.parseSequenceArg(seq_data_list[0])
for i in range(1, len(seq_data_list)):
seqs2 = self.parseSequenceArg(seq_data_list[i])
self.assertEqual(sorted(seqs1.keys()),
sorted(seqs2.keys()))
def assertSameSequences(self, seq_data_list):
seqs1 = self.parseSequenceArg(seq_data_list[0])
sd1 = self.remove_gaps(seqs1)
for i in range(1, len(seq_data_list)):
seqs2 = self.parseSequenceArg(seq_data_list[i])
sd2 = self.remove_gaps(seqs2)
self.assertEqual(sorted(sd1.values()),
sorted(sd2.values()))
def assertSameDataSet(self, seq_data_list):
seqs1 = self.parseSequenceArg(seq_data_list[0])
sd1 = self.remove_gaps(seqs1)
for i in range(1, len(seq_data_list)):
seqs2 = self.parseSequenceArg(seq_data_list[i])
sd2 = self.remove_gaps(seqs2)
self.assertSameTaxa([sd1, sd2])
self.assertSameSequences([sd1, sd2])
for name, seq in list(sd1.items()):
self.assertEqual(seq, sd2[name])
def assertSameFiles(self, files):
all_equal = True
f1 = files.pop(0)
if isinstance(f1, str):
f1 = open(f1, 'rU')
s1 = f1.read()
for f2 in files:
if isinstance(f2, str):
f2 = open(f2, 'rU')
s2 = f2.read()
if not s1 == s2:
all_equal = False
_LOG.error('files {0!r} and {1!r} are different!'.format(
f1.name, f2.name))
self.assertTrue(all_equal)
def assertSameInputOutputSequenceData(self,
seq_data_list1, seq_data_list2):
for i in range(len(seq_data_list1)):
_LOG.debug("comparing %s to %s" % (seq_data_list1[i],
seq_data_list2[i]))
seqs1 = self.parseSequenceArg(seq_data_list1[i])
seqs2 = self.parseSequenceArg(seq_data_list2[i])
self.assertSameDataSet([seqs1, seqs2])
def assertSameConcatenatedSequences(self,
concatenated_data, seq_data_list):
concat_in = self.concatenate_sequences(sorted(seq_data_list))
concat_out = self.parseSequenceArg(concatenated_data)
sd_in = self.remove_gaps(concat_in)
sd_out = self.remove_gaps(concat_out)
self.assertSameSequences([sd_in, sd_out])
def assertNoGapColumns(self, seq_data_list):
for seq_data in seq_data_list:
sd = self.parseSequenceArg(seq_data)
columns_to_taxa = {}
for name, seq in list(sd.items()):
for column_index, residue in enumerate(seq):
if residue == '-':
if column_index not in list(columns_to_taxa.keys()):
columns_to_taxa[column_index] = [name]
else:
columns_to_taxa[column_index].append(name)
self.assertEqual(len(list(columns_to_taxa.keys())), len(set(columns_to_taxa.keys())))
for col, name_list in list(columns_to_taxa.items()):
self.assertEqual(len(name_list), len(set(name_list)))
self.assertNotEqual(len(name_list), len(list(sd.keys())))
def random_id(self, length=8,
char_pool=string.ascii_letters + string.digits):
return ''.join(random.choice(char_pool) for i in range(length))
def get_subdir(self, name, parent_dir=None):
if not parent_dir:
parent_dir = self.ts.top_level_temp
if not parent_dir in self.dirs:
raise Exception('{0!r} is not a registered test dir. '
'you should not create a test dir outside of the '
'unit test temp file system'.format(parent_dir))
d = self.ts.create_temp_subdir(
parent=parent_dir,
prefix=self.job_name + name)
self.dirs.add(d)
self.register_dir(d)
return d
def get_path(self, name, parent_dir=None):
if not parent_dir:
parent_dir = self.ts.top_level_temp
if not parent_dir in self.dirs:
raise Exception('{0!r} is not a registered test dir. '
'you should not create a test file outside of the '
'unit test temp file system'.format(parent_dir))
p = os.path.join(parent_dir, self.job_name + name)
self.paths.add(p)
return p
def register_file(self, path):
shared = os.path.commonprefix([self.ts.top_level_temp, path])
if not shared == self.ts.top_level_temp:
raise Exception('cannot register file outside of the unit test '
'temp file system')
self.paths.add(path)
def register_dir(self, d):
self.ts._directories_created_lock.acquire()
self.ts._directories_created.add(d)
self.ts._directories_created_lock.release()
def dir_registered(self, d):
self.ts._directories_created_lock.acquire()
b = d in self.ts._directories_created
self.ts._directories_created_lock.release()
return b
def register_files(self):
_LOG.debug('registering temp file system...')
self.ts.run_generated_filenames.extend(self.paths)
for d in self.dirs:
self.register_dir(d)
self.register_dir(os.path.join(
self.ts.top_level_temp, self.job_name))
for path, dirs, files in os.walk(self.ts.top_level_temp):
for f in files:
if f.startswith(self.job_name):
self.ts.run_generated_filenames.append(
os.path.join(path, f))
elif os.path.basename(f) in self.ts.run_generated_filenames:
self.ts.run_generated_filenames.append(
os.path.join(path, f))
else:
if not f in self.ts.run_generated_filenames:
_LOG.warning('could not register {0!r}'.format(
os.path.join(path, f)))
for d in dirs:
self.register_dir(os.path.join(path, d))
def remove_dir(self, d=None):
if not d:
d = self.ts.top_level_temp
_LOG.debug('removing {0!r}...'.format(d))
self.ts.remove_dir(d)
def convert_rna_to_dna(self, seqs, reverse=False):
seq_dict = self.parseSequenceArg(seqs)
d = {}
for taxon, seq in list(seq_dict.items()):
if reverse:
d[taxon] = seq.replace('T', 'U')
else:
d[taxon] = seq.replace('U', 'T')
return d
def assert_is_nuc(self, seqs, datatype):
seq_dict = self.parseSequenceArg(seqs)
has_u = False
has_t = False
for taxon, seq in list(seq_dict.items()):
if 'U' in seq:
has_u = True
if 'T' in seq:
has_t = True
if datatype.upper() == 'DNA':
self.assertTrue(has_t)
self.assertFalse(has_u)
elif datatype.upper() == 'RNA':
self.assertTrue(has_u)
self.assertFalse(has_t)
class TempFSTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def testCreate(self):
self.assertTrue(True)
def testNoTopLevel(self):
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
cur_parent = os.path.abspath(os.curdir)
self.assertRaises(ValueError,
self.ts.create_temp_subdir,
prefix='bogus',
parent=cur_parent)
self.assertEqual(self.ts.get_remaining_directories(), set())
def testBadParForTop(self):
fn = 'THANKS.txt'
if os.path.exists(fn) and (not os.path.isdir(fn)):
self.assertRaises(OSError,
self.ts.create_top_level_temp,
prefix='bogus',
parent=fn)
else:
_LOG.warn(
"test of create_top_level_temp with file as parent skipped because '%s' does not exist"
% fn)
bogus_par = 'bogus_par'
if os.path.exists(bogus_par):
_LOG.warn(
"test of create_top_level_temp with non existent parent skipped because '%s' exists"
% bogus_par)
else:
self.assertRaises(OSError,
self.ts.create_top_level_temp,
prefix='bogus',
parent=bogus_par)
def testTop(self):
d = self.ts.create_top_level_temp(prefix='bogus 4 testing',
parent=os.curdir)
self.assertEqual(os.path.realpath(d), d)
self.assertEqual(self.ts.top_level_temp, d)
self.assertEqual(os.path.abspath(d), d)
self.assertTrue(os.path.exists(d))
self.assertTrue(os.path.isdir(d))
self.assertTrue(d in self.ts.get_remaining_directories())
# there can be only on top
self.assertRaises(AssertionError,
self.ts.create_top_level_temp,
prefix='bogus_for_testing',
parent=os.curdir)
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError, self.ts.create_subdir, 'bogus')
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError,
self.ts.create_temp_subdir,
prefix='bogus',
parent=os.curdir)
# but be created inside
ssd = os.path.join(d, 'bogussd')
sd = self.ts.create_subdir(ssd)
self.assertEqual(sd, ssd)
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(sd))
self.assertTrue(os.path.isdir(sd))
nssd = os.path.join(ssd, 'nested')
nsd = self.ts.create_subdir(nssd)
self.assertEqual(nsd, nssd)
self.assertTrue(nsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(nsd))
self.assertTrue(os.path.isdir(nsd))
tsd = self.ts.create_temp_subdir(prefix='bogus', parent=d)
self.assertEqual(os.path.realpath(tsd), tsd)
self.assertEqual(os.path.abspath(tsd), tsd)
self.assertTrue(os.path.exists(tsd))
self.assertTrue(os.path.isdir(tsd))
self.assertTrue(tsd in self.ts.get_remaining_directories())
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertEqual(len(self.ts.get_remaining_directories()), 4)
# create tempdir in nested
tnsd = self.ts.create_temp_subdir(prefix='tempinnested', parent=nsd)
self.assertEqual(os.path.realpath(tnsd), tnsd)
self.assertEqual(os.path.abspath(tnsd), tnsd)
self.assertTrue(os.path.exists(tnsd))
self.assertTrue(os.path.isdir(tnsd))
self.assertTrue(tnsd in self.ts.get_remaining_directories())
# subdirectories within create_temp_subdir should work...
innermost = os.path.join(tnsd, 'innermost')
innermostsd = self.ts.create_subdir(innermost)
self.assertEqual(innermostsd, innermost)
self.assertTrue(innermostsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(innermostsd))
self.assertTrue(os.path.isdir(innermostsd))
self.assertRaises(ValueError, self.ts.remove_dir, 'THANKS.txt')
self.assertEqual(self.ts.remove_dir(sd), True)
self.assertFalse(os.path.exists(sd))
self.assertFalse(os.path.exists(innermostsd))
# removing sd will remove nsd (because it is inside sd), so
# trying to created directories in that location should fail
self.assertRaises(OSError,
self.ts.create_temp_subdir,
prefix='tempinnested',
parent=nsd)
self.assertTrue(not os.path.exists(sd))
self.assertTrue(os.path.exists(tsd))
self.assertEqual(self.ts.remove_dir(d), True)
self.assertFalse(os.path.exists(tsd))
self.assertRaises(ValueError, self.ts.remove_dir, tsd)
self.assertRaises(ValueError, self.ts.remove_dir, d)
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
self.assertRaises(OSError,
self.ts.create_temp_subdir,
prefix='bogus',
parent=d)
def setUp(self):
self.ts = TempFS()
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting',
parent=os.curdir)
class AlignerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting',
parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_aligner(self, name):
try:
return config.create_aligner(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_aligner_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_aligner(self, name, fn):
filename = data_source_path(fn)
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
aln = self.get_aligner('%s' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('%s.%s' % (name, fn))
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
_LOG.debug('Checking results from %s against %s' %
(name, reference_fn))
if reference_aln != a:
i = 1
while True:
nrfn = reference_fn + '.' + str(i)
if os.path.exists(nrfn):
reference_aln = Alignment()
reference_aln.read_filepath(nrfn, 'FASTA')
_LOG.debug('Checking results from %s against %s' %
(name, nrfn))
if reference_aln == a:
self.assertEquals(reference_aln, a)
return True
i += 1
else:
self.assertEquals(reference_aln, a)
def testClustalW2(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('clustalw2', 'anolis.fasta')
def testOpal(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('opal', 'anolis.fasta')
def testMafft(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('mafft', 'anolis.fasta')
class TempFSTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def testCreate(self):
self.assertTrue(True)
def testNoTopLevel(self):
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
cur_parent = os.path.abspath(os.curdir)
self.assertRaises(ValueError, self.ts.create_temp_subdir, prefix='bogus', parent=cur_parent)
self.assertEquals(self.ts.get_remaining_directories(), set())
def testBadParForTop(self):
fn = 'THANKS.txt'
if os.path.exists(fn) and (not os.path.isdir(fn)):
self.assertRaises(OSError, self.ts.create_top_level_temp, prefix='bogus', parent=fn)
else:
_LOG.warn("test of create_top_level_temp with file as parent skipped because '%s' does not exist" % fn)
bogus_par = 'bogus_par'
if os.path.exists(bogus_par):
_LOG.warn("test of create_top_level_temp with non existent parent skipped because '%s' exists" % bogus_par)
else:
self.assertRaises(OSError, self.ts.create_top_level_temp, prefix='bogus', parent=bogus_par)
def testTop(self):
d = self.ts.create_top_level_temp(prefix='bogus 4 testing', parent=os.curdir)
self.assertEquals(os.path.realpath(d), d)
self.assertEquals(self.ts.top_level_temp, d)
self.assertEquals(os.path.abspath(d), d)
self.assertTrue(os.path.exists(d))
self.assertTrue(os.path.isdir(d))
self.assertTrue(d in self.ts.get_remaining_directories())
# there can be only on top
self.assertRaises(AssertionError, self.ts.create_top_level_temp, prefix='bogus_for_testing', parent=os.curdir)
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError, self.ts.create_subdir, 'bogus')
# subdirectories cannot be created outside of the top...
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='bogus', parent=os.curdir)
# but be created inside
ssd = os.path.join(d, 'bogussd')
sd = self.ts.create_subdir(ssd)
self.assertEquals(sd, ssd)
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(sd))
self.assertTrue(os.path.isdir(sd))
nssd = os.path.join(ssd, 'nested')
nsd = self.ts.create_subdir(nssd)
self.assertEquals(nsd, nssd)
self.assertTrue(nsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(nsd))
self.assertTrue(os.path.isdir(nsd))
tsd = self.ts.create_temp_subdir(prefix='bogus', parent=d)
self.assertEquals(os.path.realpath(tsd), tsd)
self.assertEquals(os.path.abspath(tsd), tsd)
self.assertTrue(os.path.exists(tsd))
self.assertTrue(os.path.isdir(tsd))
self.assertTrue(tsd in self.ts.get_remaining_directories())
self.assertTrue(sd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertEquals(len(self.ts.get_remaining_directories()), 4)
# create tempdir in nested
tnsd = self.ts.create_temp_subdir(prefix='tempinnested', parent=nsd)
self.assertEquals(os.path.realpath(tnsd), tnsd)
self.assertEquals(os.path.abspath(tnsd), tnsd)
self.assertTrue(os.path.exists(tnsd))
self.assertTrue(os.path.isdir(tnsd))
self.assertTrue(tnsd in self.ts.get_remaining_directories())
# subdirectories within create_temp_subdir should work...
innermost = os.path.join(tnsd, 'innermost')
innermostsd = self.ts.create_subdir(innermost)
self.assertEquals(innermostsd, innermost)
self.assertTrue(innermostsd in self.ts.get_remaining_directories())
self.assertTrue(d in self.ts.get_remaining_directories())
self.assertTrue(os.path.exists(innermostsd))
self.assertTrue(os.path.isdir(innermostsd))
self.assertRaises(ValueError, self.ts.remove_dir, 'THANKS.txt')
self.assertEquals(self.ts.remove_dir(sd), True)
self.assertFalse(os.path.exists(sd))
self.assertFalse(os.path.exists(innermostsd))
# removing sd will remove nsd (because it is inside sd), so
# trying to created directories in that location should fail
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='tempinnested', parent=nsd)
self.assertTrue(not os.path.exists(sd))
self.assertTrue(os.path.exists(tsd))
self.assertEquals(self.ts.remove_dir(d), True)
self.assertFalse(os.path.exists(tsd))
self.assertRaises(ValueError, self.ts.remove_dir, tsd)
self.assertRaises(ValueError, self.ts.remove_dir, d)
self.assertRaises(ValueError, self.ts.create_subdir, 'bogus')
self.assertRaises(OSError, self.ts.create_temp_subdir, prefix='bogus', parent=d)
class AlignerTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting', parent=os.curdir)
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
self.ts.remove_dir(dir)
def get_aligner(self, name):
try:
return config.create_aligner(name=name, temp_fs=self.ts)
except RuntimeError:
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_aligner_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_aligner(self, name, fn):
filename = data_source_path(fn)
alignment = Alignment()
alignment.read_filepath(filename, 'FASTA')
aln = self.get_aligner('%s' % name)
if aln is None:
_LOG.warn("test%s skipped" % name)
return
a = aln.run(alignment,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
reference_fn = data_source_path('%s.%s' % (name, fn))
reference_aln = Alignment()
reference_aln.read_filepath(reference_fn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, reference_fn))
if reference_aln != a:
i = 1
while True:
nrfn = reference_fn + '.' + str(i)
if os.path.exists(nrfn):
reference_aln = Alignment()
reference_aln.read_filepath(nrfn, 'FASTA')
_LOG.debug('Checking results from %s against %s' % (name, nrfn))
if reference_aln == a:
self.assertEquals(reference_aln, a)
return True
i += 1
else:
self.assertEquals(reference_aln, a)
def testClustalW2(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('clustalw2', 'anolis.fasta')
def testOpal(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('opal', 'anolis.fasta')
def testMafft(self):
if is_test_enabled(TestLevel.EXHAUSTIVE, _LOG):
self._impl_test_aligner('mafft', 'anolis.fasta')
class TreeEstimatorTest(unittest.TestCase):
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='treeEstimatorTest', parent=os.curdir)
self.filename = data_source_path('mafft.anolis.fasta')
self.alignment = Alignment()
self.alignment.read_filepath(data_source_path('mafft.anolis.fasta'),
'FASTA')
def tearDown(self):
dir_list = self.ts.get_remaining_directories()
for dir in dir_list:
try:
self.ts.remove_dir(dir)
except ValueError:
pass
def get_tree_estimator(self, name):
try:
return config.create_tree_estimator(name=name, temp_fs=self.ts)
except RuntimeError:
log_exception(_LOG)
_LOG.warn("""Could not create an aligner of type %s !
This could indicate a bug in create_tree_estimator_using_config() or could mean that
your installation is not configured to run this tool.
""" % name)
return None
def _impl_test_tree_estimator(self, name, datatype, partitions, **kwargs):
num_cpus = kwargs.get('num_cpus', None)
filename = data_source_path('anolis.fasta')
md = MultiLocusDataset()
md.read_files(seq_filename_list=[filename],
datatype=datatype)
md.relabel_for_pasta()
# alignment = Alignment()
# alignment.read_filepath(filename, 'FASTA')
te = self.get_tree_estimator(name)
if te is None:
_LOG.warn("test%s skipped" % name)
return
# alignment.datatype = datatype
if num_cpus:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True,
num_cpus=num_cpus)
else:
a = te.run(alignment=md,
partitions=partitions,
tmp_dir_par=self.ts.top_level_temp,
delete_temps=True)
def testRaxml(self):
# if is_test_enabled(TestLevel.SLOW, _LOG):
self._impl_test_tree_estimator('raxml', datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTree(self):
config.fasttree.add_option('options', StringUserSetting(
name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree', datatype="DNA",
partitions=[("DNA", 1, 1456)])
def testFastTreeMP(self):
config.fasttree.add_option('options', StringUserSetting(
name='options',
default='',
short_name=None,
help='Options to be passed to FastTree.',
subcategory=None))
self._impl_test_tree_estimator('fasttree', datatype="DNA",
partitions=[("DNA", 1, 1456)], num_cpus=2)
def setUp(self):
self.ts = TempFS()
self.ts.create_top_level_temp(prefix='alignerTesting', parent=os.curdir)