def _load_matrix(model):
probs, pi = np.zeros((20, 20)), np.zeros((20, ))
model = utilities.modeling(model).name
if model.lower() in ('jtt', 'jones'):
handle = StringIO(models['jtt'])
model = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'ProtParCon', 'data', 'jtt')
else:
if os.path.isfile(model):
handle = open(model)
else:
error('Unsupported model for computing expected changes, '
'calculation aborted.')
return None, None
for line in handle:
fields = line.strip().split()
if len(fields) == 20 and all(
[i.replace('.', '').isdigit() for i in fields]):
pi = np.array([float(i) for i in fields])
break
n = 0
for line in handle:
fields = line.strip().split()
if len(fields) == 20 and all(
[i.replace('.', '').isdigit() for i in fields]):
probs[n, :] = [float(field) for field in fields]
n += 1
handle.close()
return probs, pi
def asr(exe, msa, tree, model, gamma=4, alpha=1.8, freq='',
outfile='', verbose=False):
"""
General use function for (marginal) ancestral states reconstruction (ASR).
:param exe: str, path to the executable of an ASR program.
:param msa: str, path to the MSA file (must in FASTA format).
:param tree: str, path to the tree file (must in NEWICK format) or a NEWICK
format tree string (must start with "(" and end with ";").
:param model: str, substitution model for ASR. Either a path to a model
file or a valid model string (name of an empirical model plus some
other options like gamma category and equilibrium frequency option).
If a model file is in use, the file format of the model file depends
on the ASR program, see the its documentation for details.
:param gamma: int, The number of categories for the discrete gamma rate
heterogeneity model. Without setting gamma, RAxML will use CAT model
instead, while CODEML will use 4 gamma categories.
:param freq: str, the base frequencies of the twenty amino acids.
Accept empirical, or estimate, where empirical will set frequencies
use the empirical values associated with the specified substitution
model, and estimate will use a ML estimate of base frequencies.
:param alpha: float, the shape (alpha) for the gamma rate heterogeneity.
:param outfile: str, path to the output file. Whiteout setting, results
of ancestral states reconstruction will be saved using the filename
`[basename].[asrer].tsv`, where basename is the filename of MSA file
without known FASTA file extension, asrer is the name of the ASR
program (in lower case). The first line of the file will start with
'#TREE' and followed by a TAB (\t) and then a NEWICK formatted tree
string, the internal nodes were labeled. The second line of the tsv
file is intentionally left as a blank line and the rest lines of the
file are tab separated sequence IDs and amino acid sequences.
:param verbose: bool, invoke verbose or silent (default) process mode.
:return: tuple, the paths of the ancestral states file.
.. note::
If a tree (with branch lengths and/or internal nodes labeled) is
provided, the branch lengths and internal node labels) will be ignored.
If the model name combined with Gamma category numbers, i.e. JTT+G4,
WAG+G8, etc., only the name of the model will be used. For all models
contain G letter, a discrete Gamma model will be used to account for
among-site rate variation. If there is a number after letter G,
the number will be used to define number of categories in CODEML. For
RAxML, the number of categories will always be set to 4 if G presented.
"""
level = logging.INFO if verbose else logging.ERROR
logger.setLevel(level)
if os.path.isfile(msa):
msa = os.path.abspath(msa)
else:
error('Ancestral reconstruction aborted, msa {} is not a file or '
'does not exist.'.format(msa))
sys.exit(1)
tree = Tree(tree, leave=True)
if not isinstance(model, str):
error('Ancestral reconstruction aborted, model {} is not a valid '
'model name or model file.'.format(model))
sys.exit(1)
model = modeling(model)
asrer, func = _guess(exe)
if not outfile:
if msa.endswith('.trimmed.fasta'):
name = msa.replace('.trimmed.fasta', '')
else:
name = msa
outfile = '{}.{}.tsv'.format(basename(name), asrer)
if os.path.isfile(outfile):
info('Found pre-existing ancestral state file.')
else:
outfile = func(exe, msa, tree, model, gamma, alpha, freq, outfile)
return outfile
def _seqgen(exe, tree, length, freq, model, n, seed, gamma, alpha, invp,
outfile):
"""
Sequence simulation via EVOLVER.
:param exe: str, path to the executable of EVOLVER.
:param tree: str, path to the tree (must has branch lengths and in NEWICK
format).
:param length: int, the number of the amino acid sites need to be simulated.
:param freq: list or None, base frequencies of 20 amino acids.
:param model: str, name of a model a filename of a model file.
:param n: int, number of datasets (or duplicates) need to be simulated.
:param seed: int, the seed used to initiate the random number generator.
:param gamma: int, 0 means discrete Gamma model not in use, any
positive integer larger than 1 will invoke discrete Gamma model and
set the number of categories to gamma.
:param alpha: float, the Gamma shape parameter alpha, without setting, the
value will be estimated by the program, in case an initial value is
needed, the initial value of alpha will be set to 0.5.
:param invp: float, proportion of invariable site.
:param outfile: pathname of the output ML tree. If not set, default name
[basename].[program].ML.newick, where basename is the filename of the
sequence file without extension, program is the name of the ML inference
program, and newick is the extension for NEWICK format tree file.
:return: str, path to the simulation output file.
"""
wd = os.path.dirname(outfile)
cmd = [exe, '-l{}'.format(length), '-n{}'.format(n), '-z{}'.format(seed)]
m = modeling(model)
if m.type == 'custom':
try:
with open(m.name) as handle:
lines = handle.readlines()
except IndexError:
error('Invalid model file {}, Line 22 (amino acid frequencies)'
'does not exist in model file.'.format(m.name))
sys.exit(1)
r = [line.strip() for line in lines[:19]]
r = re.sub('\s+', ',', ','.join(r))
cmd.append('-r{}'.format(r))
if freq is None:
freq = re.sub(r'\s+', ',', ','.join(lines[21]))
cmd.append('-f{}'.format(freq))
freq = None
else:
cmd.append('-m{}'.format(m.name.upper()))
if freq:
cmd.append('-f{}'.format(','.join([str(i) for i in freq])))
if gamma:
cmd.append('-g{}'.format(gamma))
if alpha:
cmd.append('-a{}'.format(alpha))
cmd.extend(['-wa', '-q'])
cwd = tempfile.mkdtemp(dir=wd)
output = os.path.join(cwd, 'output.phylip')
tree = Tree(tree).file(os.path.join(cwd, 'tree.newick'))
try:
info('Simulating sequences using Seq-Gen.')
stdout, stdin = open(output, 'w'), open(tree)
process = Popen(cmd,
cwd=cwd,
stdout=stdout,
stdin=stdin,
stderr=PIPE,
universal_newlines=True)
code = process.wait()
stdout.close(), stdin.close()
if code:
msg = indent(process.stderr.read(), prefix='\t')
error('Sequence simulation via Seq-Gen failed due to:'
'\n{}'.format(tree, msg))
sys.exit(1)
else:
info('Parsing and saving simulation results.')
tree = Phylo.read(tree, 'newick')
number, nodes = tree.count_terminals(), []
for clade in tree.find_clades():
if not clade.is_terminal():
number += 1
clade.name = 'NODE{}'.format(number)
nodes.append(str(number))
try:
with open(outfile, 'w') as o:
o.write('#TREE\t{}\n'.format(tree.format('newick')))
with open(output) as f:
for line in f:
if line.strip():
i, s = line.strip().split()
if i.isdigit() and s.isdigit():
o.write('\n')
else:
if i.isdigit():
i = 'NODE{}'.format(i)
o.write('{}\t{}\n'.format(i, s))
except OSError:
error('Failed to save simulation results to {} ('
'IOError, permission denied).'.format(outfile))
outfile = ''
info('Successfully saved simulation results to {}'.format(outfile))
except OSError:
error('Invalid Seq-Gen executable {}, running Seq-Gen failed for '
'{}.'.format(exe, tree))
sys.exit(1)
finally:
shutil.rmtree(cwd)
return outfile
def _evolver(exe, tree, length, freq, model, n, seed, gamma, alpha, invp,
outfile):
"""
Sequence simulation via EVOLVER.
:param exe: str, path to the executable of EVOLVER.
:param tree: str, path to the tree (must has branch lengths and in NEWICK
format).
:param length: int, the number of the amino acid sites need to be simulated.
:param freq: list or None, base frequencies of 20 amino acids.
:param model: str, name of a model a filename of a model file.
:param n: int, number of datasets (or duplicates) need to be simulated.
:param seed: int, the seed used to initiate the random number generator.
:param gamma: int, 0 means discrete Gamma model not in use, any
positive integer larger than 1 will invoke discrete Gamma model and
set the number of categories to gamma.
:param alpha: float, the Gamma shape parameter alpha, without setting, the
value will be estimated by the program, in case an initial value is
needed, the initial value of alpha will be set to 0.5.
:param invp: float, proportion of invariable site.
:param outfile: pathname of the output ML tree. If not set, default name
[basename].[program].ML.newick, where basename is the filename of the
sequence file without extension, program is the name of the ML inference
program, and newick is the extension for NEWICK format tree file.
:return: str, path to the simulation output file.
"""
wd = os.path.dirname(outfile)
cwd = tempfile.mkdtemp(dir=wd)
dat = 'MCaa.dat'
tree = Tree(tree, leave=True)
tn, ts = tree.leaves, tree.string()
m = modeling(model)
if m.type == 'custom':
mf = m.name
else:
name = m.name
if name.lower() in models:
info('Using {} model for simulation.'.format(name))
with open(os.path.join(cwd, name), 'w') as o:
o.write(models[name.lower()])
mf = name
else:
error('PAML (evolver) does not support model {}.'.format(name))
sys.exit(1)
if freq is None:
mn = 2
f1, f2 = '', ''
else:
mn = 3
f1 = ' '.join([str(i) for i in freq[:10]])
f2 = ' '.join([str(i) for i in freq[10:]])
with open(os.path.join(cwd, dat), 'w') as o:
o.write(
MC_DAT.format(seed, tn, length, n, ts, alpha, gamma, mn, mf, f1,
f2))
try:
info('Simulating sequences using EVOLVER.')
log = os.path.join(cwd, 'simulation.log')
with open(log, 'w') as stdoe:
process = Popen([exe, '7', dat],
cwd=cwd,
stdout=stdoe,
stderr=stdoe,
universal_newlines=True)
code = process.wait()
if code:
with open(log) as handle:
error('Sequence simulation via EVOLVER failed for {} due to:'
'\n{}'.format(tree, indent(handle.read(), prefix='\t')))
sys.exit(1)
else:
info('Parsing and saving simulation results.')
simulations, tree = _evolver_parse(cwd)
try:
with open(outfile, 'w') as o:
o.write('#TREE\t{}\n'.format(tree.format('newick')))
for simulation in simulations:
o.writelines('{}\t{}\n'.format(s.id, s.seq)
for s in simulation)
o.write('\n')
except OSError:
error('Failed to save simulation results to {} ('
'IOError, permission denied).'.format(outfile))
outfile = ''
info('Successfully saved simulation results to {}'.format(outfile))
except OSError:
error('Invalid PAML (EVOLVER) executable {}, running EVOLVER failed '
'for {}.'.format(exe, tree))
sys.exit(1)
finally:
shutil.rmtree(cwd)
return outfile
def mlt(exe, msa, model='', cat=0, gamma=0, alpha=0.0, freq='empirical',
invp=0.0, start_tree='', constraint_tree='', seed=0, outfile='',
verbose=False):
"""
Common interface for inferring ML phylogenetic tree.
:param msa: str, path of the multiple sequence alignment (FASTA) file.
:param exe: str, path of the executable of the ML tree inference program.
:param model: str, name of the model or path of the model file.
:param cat: int, invoke rate heterogeneity (CAT) model and set the
number of categories to the corresponding number cat, if CAT model is
not in use, it will be ignored. When FastTree is in use, set cat=None
to invoke nocat mode.
:param gamma: int, 0 means discrete Gamma model not in use, any
positive integer larger than 1 will invoke discrete Gamma model and
set the number of categories to gamma.
:param alpha: float, the Gamma shape parameter alpha, without setting, the
value will be estimated by the program, in case an initial value is
needed, the initial value of alpha will be set to 0.5.
:param freq: str, the base frequencies of the twenty amino acids.
Accept empirical, or estimate, where empirical will set frequencies
use the empirical values associated with the specified substitution
model, and estimate will use a ML estimate of base frequencies.
:param invp: float, proportion of invariable site.
:param start_tree: str, path of the starting tree file, the tree file
must be in NEWICK format.
:param constraint_tree: str, path of the constraint tree file, the tree
file muse be in NEWICK format.
:param seed: int, the seed used to initiate the random number generator.
:param outfile: pathname of the output ML tree. If not set, default name
[basename].[program].ML.newick, where basename is the filename of the
sequence file without extension, program is the name of the ML inference
program, and newick is the extension for NEWICK format tree file.
:param verbose: bool, invoke verbose or silent process mode, default:
False, silent mode.
:return: path of the maximum-likelihood tree file.
"""
level = logging.INFO if verbose else logging.ERROR
logger.setLevel(level)
if not os.path.isfile(msa):
error('Alignment {} is not a file or does not exist.'.format(msa))
sys.exit(1)
program, func = _guess(exe)
if program == 'FastTree':
if cat in ('None', 'none', None):
cat = 0
else:
cat = 20
try:
seed = int(seed) if seed else random.randint(0, 10000)
except ValueError:
warn('Invalid seed, generating seed using random number generator.')
seed = random.randint(0, 10000)
model = modeling(model)
tree = func(exe, msa, model=model, cat=cat, gamma=gamma, alpha=alpha,
freq=freq, invp=invp, start_tree=start_tree, seed=seed,
constraint_tree=constraint_tree, outfile=outfile)
return tree
def test_model_freq_equal_gamma_invp(self):
self.assertTupleEqual(
MODEL('JTT', 'equal', 4, 0, 'estimate', 'builtin'),
modeling('JTT+FQ+G4+I'))
def test_model_gamma(self):
self.assertTupleEqual(MODEL('JTT', 'empirical', 4, 0, 0, 'builtin'),
modeling('JTT+G4'))
def test_model_freq_estimate_gamma_invp(self):
self.assertTupleEqual(
MODEL('JTT', 'estimate', 4, 0, 'estimate', 'builtin'),
modeling('JTT+I+FO+G4'))
def test_model_freq_gamma_invp_rate(self):
self.assertTupleEqual(
MODEL('JTT', 'empirical', 4, 4, 'estimate', 'builtin'),
modeling('JTT+F+G4+I+R4'))
def test_model_freq_equal(self):
self.assertTupleEqual(MODEL('JTT', 'equal', 0, 0, 0, 'builtin'),
modeling('JTT+FQ'))
def test_model_freq_estimate(self):
self.assertTupleEqual(MODEL('JTT', 'estimate', 0, 0, 0, 'builtin'),
modeling('JTT+FO'))
def test_model_freq(self):
self.assertTupleEqual(MODEL('JTT', 'empirical', 0, 0, 0, 'builtin'),
modeling('JTT+F'))
def test_model_custom(self):
self.assertTupleEqual(
MODEL((os.path.join(DATA, 'jtt')), 'empirical', 0, 0, 0, 'custom'),
modeling(os.path.join(DATA, 'jtt')))