def test_tree_internal_file_to_file(self): name = os.path.join(DATA, 'e.newick') self.assertEqual( '(A:0.20000,(B:0.30000,(C:0.40000,D:0.50000)' ':0.20000):0.10000):0.00000;', _read(Tree(self.tree_nodes).file(name))) self.rm = name
def test_tree_confidence_file_to_file(self): name = os.path.join(DATA, 'g.newick') self.assertEqual( '(A,(B,(C,D)7)6)5;', _read(Tree(self.tree_ic).file(name, ic2name=True, brlen=False))) self.rm = name
def test_tree_internal_brlen_file_to_file(self): name = os.path.join(DATA, 'f.newick') self.assertEqual('(A,(B,(C,D)));', _read(Tree(self.tree_nodes).file(name, brlen=False))) self.rm = name
def test_tree_brlen_file_to_file(self): name = os.path.join(DATA, 'd.newick') self.assertEqual( '(A,(B,(C,D)E)F)root;', _read(Tree(self.tree_nodes).file(name, brlen=False, nodes=True))) self.rm = name
def test_leave(self): with self.assertRaises(SystemExit) as cm: Tree(self.s + 'abc', leave=True) self.assertEqual(cm.exception.code, 1)
def test_topology_internal_file_to_file(self): name = os.path.join(DATA, 'b.newick') self.assertEqual('(A,(B,(C,D)));', _read(Tree(self.top_nodes).file(name))) self.rm = name
def test_tree_internal_string(self): self.assertEqual( '(A:0.20000,(B:0.30000,(C:0.40000,D:0.50000):' '0.20000):0.10000):0.00000;', Tree(self.s_nodes).string())
def sim(exe, tree='', sequence='', model='JTT', length=100, freq='empirical', n=100, seed=0, gamma=4, alpha=0.5, invp=0, outfile='', verbose=False): """ Sequence simulation via EVOLVER. :param exe: str, path to the executable of EVOLVER. :param tree: str, path to the tree (must has branch lengths and in NEWICK format). If not provided, sequence file need to be a tsv file consisting a tree with branch lengths and sequences. If both tree and sequence in tsv format file were provided, the tree in tsv file will be ignored. :param sequence: str, path to a multiple sequence alignment file in FASTA format or a tsv file generated by function `asr()` that have a line contains a tree with branch lengths. If provided, the length and base amino acid frequencies will be calculated based on the leaf sequences. :param model: str, name of a model a filename of a model file. :param length: int, the number of the amino acid sites need to be simulated, default: 0, the length will be obtained from the sequence. :param freq: str, "empirical", "estimate", or a comma separated string of base frequencies of 20 amino acids in the order of "ARNDCQEGHILKMFPSTWYV". :param n: int, number of datasets (or duplicates) need to be simulated. :param seed: int, the seed used to initiate the random number generator. :param gamma: int, 0 means discrete Gamma model not in use, any positive integer larger than 1 will invoke discrete Gamma model and set the number of categories to gamma. :param alpha: float, the Gamma shape parameter alpha, without setting, the value will be estimated by the program, in case an initial value is needed, the initial value of alpha will be set to 0.5. :param invp: float, proportion of invariable site. :param outfile: pathname of the output ML tree. If not set, default name [basename].[program].ML.newick, where basename is the filename of the sequence file without extension, program is the name of the ML inference program, and newick is the extension for NEWICK format tree file. :param verbose: bool, invoke verbose or silent process mode, default: False, silent mode. :return: str, path to the simulation output file. """ logger.setLevel(logging.INFO if verbose else logging.ERROR) if tree: t = Tree(tree, leave=True) if not t.length: error('Unscaled tree: {}, cannot simulate sequences without branch' 'lengths.'.format(tree)) sys.exit(1) if sequence: _, l, freqs = _seq2info(sequence) else: l, freqs = 0, None elif sequence: t, l, freqs = _seq2info(sequence) else: error('Neither tree or a sequence file contains a tree was provided, ' 'simulation aborted.') sys.exit(1) if length: try: length = int(length) except ValueError: error('Invalid length, length should be a integer.') sys.exit(1) else: if l: length = l else: error('Neither valid sequence nor argument length was specified, ' 'failed to obtain length, simulation aborted.') sys.exit(1) fs = freqs if freq == 'estimate' else None if freq: if freq == 'equal': fs = ['0.05'] * 20 elif freq == 'estimate': fs = freqs if not fs: warn('Failed to get observed amino acid frequency from ' 'sequence, use the default frequency of model instead.') elif freq == 'empirical': fs = None elif freq.startswith('0') and freq.count(',') == 19: fs = [i.strip() for i in freq.split(',')] if 1 - sum([float(s) for s in fs]) > 0.000001: error('Specified frequencies do not add up to 1.0, simulation ' 'aborted.') sys.exit(1) else: warn('Unknown frequency encounter, use the default frequency of ' 'model instead.') fs = None try: seed = int(seed) if seed else random.randint(0, 10000) except ValueError: warn('Invalid seed, use generated random number instead.') seed = random.randint(0, 10000) name, func = _guess(exe) if not outfile: outfile = os.path.join(os.getcwd(), '{}.simulations.tsv'.format(name)) if os.path.isfile(outfile): info('Found pre-existing simulated sequences.') else: outfile = func(exe, tree, length, fs, model, n, seed, gamma, alpha, invp, outfile) return outfile
def test_tree_string(self): self.assertEqual( '(A:0.20000,(B:0.30000,(C:0.40000,D:0.50000)' ':0.20000):0.10000):0.00000;', Tree(self.s).string())
def test_tree_brlen_string(self): self.assertEqual('(A,(B,(C,D)E)F)root;', Tree(self.s_nodes).string(brlen=False, nodes=True))
def test_topology_internal_string(self): self.assertEqual('(A,(B,(C,D)));', Tree('(A,(B,(C,D)E)F)root;').string())
def test_topology_string(self): self.assertEqual('(A,(B,(C,D)));', Tree('(A,(B,(C,D)));').string())
def test_topology_internal_file(self): self.assertEqual('(A,(B,(C,D)));', Tree(self.top_nodes).string())
def _evolver(exe, tree, length, freq, model, n, seed, gamma, alpha, invp, outfile): """ Sequence simulation via EVOLVER. :param exe: str, path to the executable of EVOLVER. :param tree: str, path to the tree (must has branch lengths and in NEWICK format). :param length: int, the number of the amino acid sites need to be simulated. :param freq: list or None, base frequencies of 20 amino acids. :param model: str, name of a model a filename of a model file. :param n: int, number of datasets (or duplicates) need to be simulated. :param seed: int, the seed used to initiate the random number generator. :param gamma: int, 0 means discrete Gamma model not in use, any positive integer larger than 1 will invoke discrete Gamma model and set the number of categories to gamma. :param alpha: float, the Gamma shape parameter alpha, without setting, the value will be estimated by the program, in case an initial value is needed, the initial value of alpha will be set to 0.5. :param invp: float, proportion of invariable site. :param outfile: pathname of the output ML tree. If not set, default name [basename].[program].ML.newick, where basename is the filename of the sequence file without extension, program is the name of the ML inference program, and newick is the extension for NEWICK format tree file. :return: str, path to the simulation output file. """ wd = os.path.dirname(outfile) cwd = tempfile.mkdtemp(dir=wd) dat = 'MCaa.dat' tree = Tree(tree, leave=True) tn, ts = tree.leaves, tree.string() m = modeling(model) if m.type == 'custom': mf = m.name else: name = m.name if name.lower() in models: info('Using {} model for simulation.'.format(name)) with open(os.path.join(cwd, name), 'w') as o: o.write(models[name.lower()]) mf = name else: error('PAML (evolver) does not support model {}.'.format(name)) sys.exit(1) if freq is None: mn = 2 f1, f2 = '', '' else: mn = 3 f1 = ' '.join([str(i) for i in freq[:10]]) f2 = ' '.join([str(i) for i in freq[10:]]) with open(os.path.join(cwd, dat), 'w') as o: o.write( MC_DAT.format(seed, tn, length, n, ts, alpha, gamma, mn, mf, f1, f2)) try: info('Simulating sequences using EVOLVER.') log = os.path.join(cwd, 'simulation.log') with open(log, 'w') as stdoe: process = Popen([exe, '7', dat], cwd=cwd, stdout=stdoe, stderr=stdoe, universal_newlines=True) code = process.wait() if code: with open(log) as handle: error('Sequence simulation via EVOLVER failed for {} due to:' '\n{}'.format(tree, indent(handle.read(), prefix='\t'))) sys.exit(1) else: info('Parsing and saving simulation results.') simulations, tree = _evolver_parse(cwd) try: with open(outfile, 'w') as o: o.write('#TREE\t{}\n'.format(tree.format('newick'))) for simulation in simulations: o.writelines('{}\t{}\n'.format(s.id, s.seq) for s in simulation) o.write('\n') except OSError: error('Failed to save simulation results to {} (' 'IOError, permission denied).'.format(outfile)) outfile = '' info('Successfully saved simulation results to {}'.format(outfile)) except OSError: error('Invalid PAML (EVOLVER) executable {}, running EVOLVER failed ' 'for {}.'.format(exe, tree)) sys.exit(1) finally: shutil.rmtree(cwd) return outfile
def test_tree_internal_brlen_string(self): self.assertEqual('(A,(B,(C,D)));', Tree(self.s_nodes).string(brlen=False))
def _seqgen(exe, tree, length, freq, model, n, seed, gamma, alpha, invp, outfile): """ Sequence simulation via EVOLVER. :param exe: str, path to the executable of EVOLVER. :param tree: str, path to the tree (must has branch lengths and in NEWICK format). :param length: int, the number of the amino acid sites need to be simulated. :param freq: list or None, base frequencies of 20 amino acids. :param model: str, name of a model a filename of a model file. :param n: int, number of datasets (or duplicates) need to be simulated. :param seed: int, the seed used to initiate the random number generator. :param gamma: int, 0 means discrete Gamma model not in use, any positive integer larger than 1 will invoke discrete Gamma model and set the number of categories to gamma. :param alpha: float, the Gamma shape parameter alpha, without setting, the value will be estimated by the program, in case an initial value is needed, the initial value of alpha will be set to 0.5. :param invp: float, proportion of invariable site. :param outfile: pathname of the output ML tree. If not set, default name [basename].[program].ML.newick, where basename is the filename of the sequence file without extension, program is the name of the ML inference program, and newick is the extension for NEWICK format tree file. :return: str, path to the simulation output file. """ wd = os.path.dirname(outfile) cmd = [exe, '-l{}'.format(length), '-n{}'.format(n), '-z{}'.format(seed)] m = modeling(model) if m.type == 'custom': try: with open(m.name) as handle: lines = handle.readlines() except IndexError: error('Invalid model file {}, Line 22 (amino acid frequencies)' 'does not exist in model file.'.format(m.name)) sys.exit(1) r = [line.strip() for line in lines[:19]] r = re.sub('\s+', ',', ','.join(r)) cmd.append('-r{}'.format(r)) if freq is None: freq = re.sub(r'\s+', ',', ','.join(lines[21])) cmd.append('-f{}'.format(freq)) freq = None else: cmd.append('-m{}'.format(m.name.upper())) if freq: cmd.append('-f{}'.format(','.join([str(i) for i in freq]))) if gamma: cmd.append('-g{}'.format(gamma)) if alpha: cmd.append('-a{}'.format(alpha)) cmd.extend(['-wa', '-q']) cwd = tempfile.mkdtemp(dir=wd) output = os.path.join(cwd, 'output.phylip') tree = Tree(tree).file(os.path.join(cwd, 'tree.newick')) try: info('Simulating sequences using Seq-Gen.') stdout, stdin = open(output, 'w'), open(tree) process = Popen(cmd, cwd=cwd, stdout=stdout, stdin=stdin, stderr=PIPE, universal_newlines=True) code = process.wait() stdout.close(), stdin.close() if code: msg = indent(process.stderr.read(), prefix='\t') error('Sequence simulation via Seq-Gen failed due to:' '\n{}'.format(tree, msg)) sys.exit(1) else: info('Parsing and saving simulation results.') tree = Phylo.read(tree, 'newick') number, nodes = tree.count_terminals(), [] for clade in tree.find_clades(): if not clade.is_terminal(): number += 1 clade.name = 'NODE{}'.format(number) nodes.append(str(number)) try: with open(outfile, 'w') as o: o.write('#TREE\t{}\n'.format(tree.format('newick'))) with open(output) as f: for line in f: if line.strip(): i, s = line.strip().split() if i.isdigit() and s.isdigit(): o.write('\n') else: if i.isdigit(): i = 'NODE{}'.format(i) o.write('{}\t{}\n'.format(i, s)) except OSError: error('Failed to save simulation results to {} (' 'IOError, permission denied).'.format(outfile)) outfile = '' info('Successfully saved simulation results to {}'.format(outfile)) except OSError: error('Invalid Seq-Gen executable {}, running Seq-Gen failed for ' '{}.'.format(exe, tree)) sys.exit(1) finally: shutil.rmtree(cwd) return outfile
def test_tree_confidence_string(self): self.assertEqual('(A,(B,(C,D)7)6)5;', Tree(self.s_ic).string(ic2name=True, brlen=False))
def asr(exe, msa, tree, model, gamma=4, alpha=1.8, freq='', outfile='', verbose=False): """ General use function for (marginal) ancestral states reconstruction (ASR). :param exe: str, path to the executable of an ASR program. :param msa: str, path to the MSA file (must in FASTA format). :param tree: str, path to the tree file (must in NEWICK format) or a NEWICK format tree string (must start with "(" and end with ";"). :param model: str, substitution model for ASR. Either a path to a model file or a valid model string (name of an empirical model plus some other options like gamma category and equilibrium frequency option). If a model file is in use, the file format of the model file depends on the ASR program, see the its documentation for details. :param gamma: int, The number of categories for the discrete gamma rate heterogeneity model. Without setting gamma, RAxML will use CAT model instead, while CODEML will use 4 gamma categories. :param freq: str, the base frequencies of the twenty amino acids. Accept empirical, or estimate, where empirical will set frequencies use the empirical values associated with the specified substitution model, and estimate will use a ML estimate of base frequencies. :param alpha: float, the shape (alpha) for the gamma rate heterogeneity. :param outfile: str, path to the output file. Whiteout setting, results of ancestral states reconstruction will be saved using the filename `[basename].[asrer].tsv`, where basename is the filename of MSA file without known FASTA file extension, asrer is the name of the ASR program (in lower case). The first line of the file will start with '#TREE' and followed by a TAB (\t) and then a NEWICK formatted tree string, the internal nodes were labeled. The second line of the tsv file is intentionally left as a blank line and the rest lines of the file are tab separated sequence IDs and amino acid sequences. :param verbose: bool, invoke verbose or silent (default) process mode. :return: tuple, the paths of the ancestral states file. .. note:: If a tree (with branch lengths and/or internal nodes labeled) is provided, the branch lengths and internal node labels) will be ignored. If the model name combined with Gamma category numbers, i.e. JTT+G4, WAG+G8, etc., only the name of the model will be used. For all models contain G letter, a discrete Gamma model will be used to account for among-site rate variation. If there is a number after letter G, the number will be used to define number of categories in CODEML. For RAxML, the number of categories will always be set to 4 if G presented. """ level = logging.INFO if verbose else logging.ERROR logger.setLevel(level) if os.path.isfile(msa): msa = os.path.abspath(msa) else: error('Ancestral reconstruction aborted, msa {} is not a file or ' 'does not exist.'.format(msa)) sys.exit(1) tree = Tree(tree, leave=True) if not isinstance(model, str): error('Ancestral reconstruction aborted, model {} is not a valid ' 'model name or model file.'.format(model)) sys.exit(1) model = modeling(model) asrer, func = _guess(exe) if not outfile: if msa.endswith('.trimmed.fasta'): name = msa.replace('.trimmed.fasta', '') else: name = msa outfile = '{}.{}.tsv'.format(basename(name), asrer) if os.path.isfile(outfile): info('Found pre-existing ancestral state file.') else: outfile = func(exe, msa, tree, model, gamma, alpha, freq, outfile) return outfile
def test_log(self): with self.assertLogs('[iMC]', level='INFO') as cm: Tree(self.s + 'abc') self.assertRegex(cm.output[0], r'ERROR.*Invalid tree:.*a NEWICK.*.')