def msa(exe, seq, outfile='', trimming=False, verbose=False): """ General use function for multiple sequence alignment (MSA). :param exe: str, path to the executable of a MSA program. :param seq: str, path to the multiple sequence file (must in FASTA format). :param outfile: str, path to the aligned sequence output (FASTA) file, default: [basename].[aligner].fasta, where basename is the filename of the sequence file without known FASTA file extension, aligner is the name of the aligner program in lowercase, and fasta is the extension for fasta format file. :param trimming: bool, trim gaps and ambiguous sites if True, otherwise, leave them untouched. :param verbose: bool, invoke verbose or silent process mode, default: False, silent mode. :return: str, path to the aligned sequence output file (in FASTA format). """ level = logging.INFO if verbose else logging.ERROR logger.setLevel(level) if os.path.isfile(seq): sequence = os.path.abspath(seq) if exe: aligner, func = _guess(exe) if func is None: error('Invalid or unsupported aligner executable (exe): {}, ' 'alignment aborted.'.format(exe)) sys.exit(1) else: error('Invalid aligner executable (exe), empty string, sequence ' 'alignment aborted.') sys.exit(1) if not outfile: outfile = '.'.join([basename(sequence), aligner, 'fasta']) if os.path.isfile(outfile): info('Found pre-existing alignment file.') else: info('Aligning sequence {} using {}.'.format(sequence, aligner.upper())) outfile = func(exe, sequence, outfile) info('Successfully aligned sequence, alignment was saved to ' '{}.'.format(outfile)) else: error('Sequence: {} is not a file or does not exist.'.format(seq)) sys.exit(1) if trimming: clean = ''.join([basename(outfile), '.trimmed.fasta']) if os.path.isfile(clean): outfile = clean info('Found pre-existing trimmed alignment.') else: _, outfile = trim(outfile, outfile=clean, verbose=verbose) return outfile
def _sequencing(sequence, tree, aligner, ancestor, wd, asr_model, verbose): """ Identify the type of the sequence file. :param sequence: str, path to a sequence data file. :param tree: str, path to a NEWICK tree file. :return: tuple, sequence, alignment, ancestor, and simulation data file. """ if tree: utilities.Tree(tree, leave=True) AA, lengths, aa = set(AMINO_ACIDS), [], [] with open(sequence) as handle: line = handle.readline().strip() if line.startswith('>'): handle.seek(0) records = SeqIO.parse(handle, 'fasta') for record in records: lengths.append(len(record.seq)) aa.append(set(record.seq).issubset(AA)) else: error('NEWICK format tree was provided, but the sequence file ' 'was not in the FASTA format.') sys.exit(1) if len(set(lengths)) == 1: alignment = sequence if all(aa): trimmed = alignment else: trimmed = ''.join( [utilities.basename(alignment), '.trimmed.fasta']) if os.path.isfile(trimmed): info('Using pre-existed trimmed alignment file.') else: _, trimmed = utilities.trim(alignment, outfile=trimmed) else: if aligner: aler, _ = msa._guess(aligner) outfile = ''.join( [utilities.basename(sequence), '.{}.fasta'.format(aler)]) if os.path.isfile(outfile): info('Using pre-existed alignment file') alignment = outfile trimmed = ''.join( [utilities.basename(alignment), '.trimmed.fasta']) if os.path.isfile(trimmed): info('Using pre-existed trimmed alignment file.') else: _, trimmed = utilities.trim(alignment, outfile=trimmed) else: trimmed = msa.msa(aligner, sequence, verbose=verbose, outfile=outfile, trimming=True) else: error('FASTA format sequence file was provided, but no ' 'alignment program was provided.') sys.exit(1) if trimmed: if ancestor: if trimmed.endswith('.trimmed.fasta'): name = trimmed.replace('.trimmed.fasta', '') else: name = trimmed aser, _ = asr._guess(ancestor) outfile = '{}.{}.tsv'.format(utilities.basename(name), aser) if os.path.isfile(outfile): info('Using pre-existed ancestral states sequence file.') sequence = outfile else: sequence = asr.asr(ancestor, trimmed, tree, asr_model, verbose=verbose, outfile=outfile) else: error('No ancestral reconstruction program was provided.') sys.exit(1) else: sys.exit(1) tree, rate, records, aps, size = _load(sequence) return tree, rate, records, aps, size, sequence
def test_tsv(self): self.assertEqual('a.name', basename('a.name.tsv'))
def test_fasta(self): self.assertEqual('a.name', basename('a.name.fasta'))
def test_text(self): self.assertEqual('a.name', basename('a.name.text'))
def _iqtree(exe, msa, model, cat, gamma, alpha, freq, invp, start_tree, constraint_tree, seed, outfile): """ Infer ML phylogenetic tree using IQ-TREE. """ wd = tempfile.mkdtemp(dir=os.path.dirname(os.path.abspath(msa))) shutil.copy(msa, os.path.join(wd, 'msa.fa')) if model.name: name = model.name.upper() if model.type == 'builtin': frequency = freq or model.frequency if frequency == 'estimate': frequency = 'FO' else: frequency = 'F' invpro = 'I' if (invp or model.invp) else '' gamma = gamma or model.gamma gamma = 'G{}'.format(gamma) if gamma else '' cat = cat if cat not in ('None', 'none', None) else 0 rates = cat or model.rates rates = 'R{}'.format(rates) if rates else '' m = ['-m', '+'.join((i for i in [name, frequency, invpro, gamma, rates] if i))] else: m = ['-mdef', model.name] else: m = ['-m', 'TEST'] info('Inferring ML tree for {} using IQ-TREE.'.format(msa)) # iqtree_cmd = 'exe -s seq -t user_tree -pre prefix -nt 1 -seed seed # -quiet -m MFP -g constraint_tree' args = [exe, '-s', 'msa.fa', '-nt', '1', '-pre', 'tmf', '-seed', str(seed)] args.extend(m) # args.append('-quiet') if gamma and alpha: args.extend(['-a', str(alpha)]) if invp: args.extend(['-i', str(invp)]) if constraint_tree: args.extend(['-g', constraint_tree]) try: # info('Running IQTree using the following command:\n\t' # '{}'.format(' '.join(args))) process = Popen(args, cwd=wd, stdout=PIPE, stderr=PIPE, universal_newlines=True) code = process.wait() if code: msg = indent(process.stderr.read(), prefix='\t') error('Inferring ML tree failed for {}\n{}'.format(msa, msg)) sys.exit(1) else: tree = outfile if outfile else '{}.IQ-TREE.ML.newick'.format( basename(msa)) try: tree = shutil.copy(os.path.join(wd, 'tmf.treefile'), tree) info('Successfully save inferred gene tree to {}.'.format( tree)) except OSError: error('Path of outfile {} is not writeable, saving tree to ' 'file failed.'.format(tree)) sys.exit(1) except OSError: error('Inferring tree failed for {}, executable (exe) {} of ' 'IQ-TREE is invalid.'.format(msa, exe)) sys.exit(1) finally: shutil.rmtree(wd) return tree
def _phyml(exe, msa, model, cat, gamma, alpha, freq, invp, start_tree, constraint_tree, seed, outfile): """ Infer ML phylogenetic tree using PhyML. """ # cmd = 'exe -i seq -d aa -m JTT -f e|m -v invariable -c 4 -a gamma-alpha # --quiet --r_seed num -u user_tree_file' wd = tempfile.mkdtemp(dir=os.path.dirname(os.path.abspath(msa))) alignment = 'temporary.alignment.phylip' AlignIO.convert(msa, 'fasta', os.path.join(wd, alignment), 'phylip') if model.type == 'builtin': m = ['-m', model.name] if model.name else ['-m', 'LG'] else: m = ['-m', 'custom', '--aa_rate_file', model.name] info('Inferring ML tree for {} using PhyML.'.format(msa)) args = [exe, '-i', alignment, '-d', 'aa', '--r_seed', str(seed), '--quiet'] args.extend(m) cat = cat if cat not in ('None', 'none', None) else 0 if cat: args.extend(['--free_rates', cat]) gamma = gamma or model.gamma if gamma: args.extend(['-c', str(gamma)]) if alpha: args.extend(['-a', str(alpha)]) frequency = freq or model.frequency frequency = 'X' if frequency == 'estimate' else 'F' if frequency == 'estimate': args.extend(['-f', 'e']) else: args.extend(['-f', 'm']) if start_tree: args.extend(['-u', start_tree]) if constraint_tree: args.extend(['--constraint_file', constraint_tree]) if model.invp: if invp: args.extend(['-v', str(invp)]) else: args.extend(['-v', 'e']) else: if invp: args.extend(['-v', str(invp)]) try: # info('Running FastTree using the following command:\n\t' # '{}'.format(' '.join(args))) process = Popen(args, cwd=wd, stdout=PIPE, stderr=PIPE, universal_newlines=True) code = process.wait() if code: msg = process.stderr.read() or process.stdout.read() msg = indent(msg, prefix='\t') error('Tree inferring failed for {}\n{}'.format(msa, msg)) sys.exit(1) else: tree = outfile if outfile else '{}.PhyML.ML.newick'.format( basename(msa)) try: out = '{}{}'.format(os.path.join(wd, alignment), '_phyml_tree.txt') tree = shutil.copy(out, tree) info('Successfully save inferred ML tree to {}.'.format( tree)) except OSError: error('Path of outfile {} is not writeable, saving tree to ' 'file failed.'.format(tree)) sys.exit(1) except OSError: error('Tree inferring failed for {}, executable (exe) {} of PhyML ' 'is invalid.'.format(msa, exe)) sys.exit(1) finally: shutil.rmtree(wd) return tree
def test_stockholm(self): self.assertEqual('a.name', basename('a.name.stockholm'))
def test_emb(self): self.assertEqual('a.name', basename('a.name.emb'))
def test_phylip_relaxed(self): self.assertEqual('a.name', basename('a.name.phylip-relaxed'))
def test_phy_sequential(self): self.assertEqual('a.name', basename('a.name.phylip-sequential'))
def test_phylip(self): self.assertEqual('a.name', basename('a.name.phylip'))
def test_clustal(self): self.assertEqual('a.name', basename('a.name.clustal'))
def test_aln(self): self.assertEqual('a.name', basename('a.name.aln'))
def imc(sequence, tree='', aligner='', ancestor='', simulator='', asr_model='JTT', exp_model='JTT', n=100, divergent=True, indpairs=True, threshold=0.0, exp_prob=False, verbose=False): """ Identify molecular parallel and convergent changes. :param sequence: str, path to the sequence data file. Sequence data file here covers a wide range of files and formats: * sequences: raw protein sequence file, need to be in FASTA format and a NEWICK format tree is also required for argument tree. * msa: multiple sequence alignment file, need to be in FASTA format and a NEWICK format tree is also required for argument tree. * ancestors: reconstructed ancestral states file, need to be in tsv (tab separated) file, the first line needs to start with #TREE, second line needs to be a blank line, and the rest lines in the file need to be tab separated sequence name (or ID) and amino acid sequences. * simulations: simulated sequences, need to be in tsv file, the first line needs to start with #TREE, second line needs to be a blank line, each dataset need to be separated by a blank line and inside each dataset block, each line should consist of tab separated sequence name (or ID) and amino acid sequences. :param tree: str, NEWICK format tree string or tree file. This need to be set according to argument sequence. if sequence is raw sequence file or MSA file, tree is required for guiding ancestral states reconstruction. If sequence is ancestors or simulations, then tree is not necessary. :param aligner: str, executable of an alignment program. :param ancestor: str, executable of an ancestral states reconstruction program. :param simulator: str, executable of an sequence simulation program. :param asr_model: str, model name or model file for ancestral states reconstruction, default: JTT. :param exp_model: str, model name or model file for estimate expected changes based on simulation or replacement probability manipulation, default: JTT. :param n: int, number of datasets (or duplicates) should be simulated. :param divergent: bool, identify divergent changes if True, or only identify parallel and convergent changes if False. :param indpairs: bool, only identify changes for independent branch pairs if true, or identify changes for all branch pairs if False. :param threshold: float, a probability threshold that ranges from 0.0 to 1.0. If provided, only ancestral states with probability equal or larger than the threshold will be used, default: 0.0. :param exp_prob: bool, calculate the probability of expected changes if set to True and the exp_model contains a probability matrix. Time consuming process, be patient for the calculation. :param verbose: bool, invoke verbose or silent process mode, default: False, silent mode. :return: tuple, a dict object of counts of parallel replacements, a dict object of counts of convergent replacements, a list consists of details of replacements (namedtuple) and the p-value of AU Test (float or None). """ logger.setLevel(logging.INFO if verbose else logging.ERROR) if os.path.isfile(sequence): sequence = os.path.abspath(sequence) wd = os.path.dirname(sequence) else: error('Invalid sequence {}, sequence is not a file or dose not ' 'exist, exited.'.format(sequence)) sys.exit(1) basename = utilities.basename(sequence) rs = _sequencing(sequence, tree, aligner, ancestor, wd, asr_model, verbose) tree, rates, records, aps, size, sequence = rs basename_more = utilities.basename(sequence) pars, cons, divs, details, aup = None, None, None, None, None h1 = ['Category', 'BranchPair'] h2 = ['Category', 'Position', 'BranchPair', 'R1', 'R2', 'Dataset'] probs, pi = None, None if size == 1: h1.append('OBS') if exp_model: if simulator: h1.append('EXP') h1.extend(['SIM-{}'.format(i + 1) for i in range(n)]) else: if exp_prob: probs, pi = _load_matrix(exp_model) if probs is not None: h1.append('EXP') else: h1.append('EXP') h1.extend(['SIM-{}'.format(i + 1) for i in range(size)]) tips = [v[0] for k, v in records.items() if not k.startswith('NODE')] length = len(tips[0]) if size > 1: info('Estimating expected changes ... ') else: info('Identifying observed changes ...') tree, pars, cons, divs, details = _pc(tree, rates, records, aps, size, length, probs, pi, indpairs, threshold) if size == 1 and simulator: freq = _frequencing(tips, site=False) ts = tree.format('newick').strip() out = '{}.{}.tsv'.format(basename, sim._guess(simulator)[0]) s = sim.sim(simulator, ts, model=exp_model, length=length, freq=freq, n=n, outfile=out, verbose=verbose) if s and os.path.isfile(s): tree, rates, records, aps, size = _load(s) info('Estimating expected changes ... ') tree, par, con, div, detail = _pc(tree, rates, records, aps, size, length, None, None, indpairs, threshold) for k, v in par.items(): pars[k].append(np.mean(v)) cons[k].append(np.mean(con[k])) divs[k].append(np.mean(div[k])) pars[k].extend(v), cons[k].extend(con[k]) divs[k].extend(div[k]) details.extend(detail) if any([pars, cons, divs, details]): info('Writing identified parallel and convergent amino acid ' 'replacements to files.') counts = ''.join([basename_more, '.counts.tsv']) changes = ''.join([basename_more, '.details.tsv']) with open(counts, 'w') as o, open(changes, 'w') as c: o.write('{}\n'.format('\t'.join(h1))) s = lambda x: '{:.4f}'.format(x) if isinstance(x, float) else str(x ) o.writelines('P\t{}\t{}\n'.format(k, '\t'.join([s(x) for x in v])) for k, v in pars.items()) o.writelines('C\t{}\t{}\n'.format(k, '\t'.join([s(x) for x in v])) for k, v in cons.items()) o.writelines('D\t{}\t{}\n'.format(k, '\t'.join([s(x) for x in v])) for k, v in divs.items()) c.write('{}\n'.format('\t'.join(h2))) c.writelines('{}\t{}\t{}\t{}\t{}\t{}\n'.format(*detail) for detail in details) return pars, cons, divs, details, length
def test_maf(self): self.assertEqual('a.name', basename('a.name.maf'))
def _fasttree(exe, msa, model, cat, gamma, alpha, freq, invp, start_tree, constraint_tree, seed, outfile): """ Infer ML phylogenetic tree using FastTree. """ if model.type == 'custom': error('Invalid model, FastTree does not accept custom model file ' '{}.'.format(model)) sys.exit(1) else: name = model.name if model.name else 'JTT' if name.lower() == 'jtt': info('Inferring ML tree using JTT model.') m = '' else: if name.lower in ['lg', 'wag']: info('Inferring ML tree using {} model.'.format( model.upper())) m = '-{}'.format(name) else: error('Invalid model {}, FastTree only accepts JTT, WAG, ' 'and LG model.'.format(name)) sys.exit(1) info('Inferring ML tree for {} using FastTree.'.format(msa)) args = [exe, '-quiet', '-nopr', '-seed', str(seed)] if start_tree and os.path.isfile(start_tree): args.extend(['-intree', start_tree]) if m: args.append(m) if gamma or model.gamma: args.append('-gamma') if cat: if cat != 20: args.extend(['-cat', str(cat)]) else: args.append('-nocat') args.append(os.path.basename(msa)) tree = outfile if outfile else '{}.FastTree.ML.newick'.format(basename(msa)) try: info('Running FastTree using the following command:\n\t' '{}'.format(' '.join(args))) with open(tree, 'w') as stdout: process = Popen(args, cwd=os.path.dirname(msa), stdout=stdout, stderr=PIPE, universal_newlines=True) code = process.wait() if code: msg = indent(process.stderr.read(), prefix='\t') print(msg) error('Inferring ML tree failed for {}\n{}'.format(msa, msg)) if os.path.isfile(tree): os.remove(tree) sys.exit(1) else: info('Successfully save inferred ML tree to {}.'.format(tree)) except OSError: error('Inferring ML tree failed for {}, executable (exe) {} of ' 'FastTree is empty or invalid.'.format(msa, exe)) if os.path.isfile(tree): os.remove(tree) sys.exit(1) return tree
def test_xmfa(self): self.assertEqual('a.name', basename('a.name.xmfa'))
def _raxml(exe, msa, model, cat, gamma, alpha, freq, invp, start_tree, constraint_tree, seed, outfile): """ Infer ML phylogenetic tree using RAxML. """ wd = tempfile.mkdtemp(dir=os.path.dirname(os.path.abspath(msa))) shutil.copy(msa, os.path.join(wd, 'msa.fa')) if model.type == 'builtin': frequency = freq or model.frequency frequency = 'X' if frequency == 'estimate' else 'F' gamma = gamma or model.gamma invpro = 'I' if (model.invp or invp) else '' gamma = 'GAMMA' if gamma else 'CAT' if model.name: mm = ''.join(['PROT', gamma, invpro, model.name.upper(), frequency]) model = ['-m', mm] else: model = ['-m', ''.join(['PROT', invpro, gamma, 'AUTO'])] else: model = ['-P', model.name] info('Inferring ML tree for {} using RAxML.'.format(msa)) args = [exe, '-s', 'msa.fa', '-n', 'RAxML', '-p', str(seed), '--silent'] args.extend(model) cat = cat if cat not in ('None', 'none', None) else 0 if cat: args.extend(['-c', str(cat)]) if start_tree: args.extend(['-t', start_tree]) if constraint_tree: args.extend(['-r', constraint_tree]) try: # info('Running FastTree using the following command:\n\t' # '{}'.format(' '.join(args))) process = Popen(args, cwd=wd, stdout=DEVNULL, stderr=PIPE, universal_newlines=True) code = process.wait() if code: msg = indent(process.stderr.read(), prefix='\t') error('Tree inferring failed for {}\n{}'.format(msa, msg)) tree = '' else: tree = outfile if outfile else '{}.RAxML.ML.newick'.format( basename(msa)) try: tree = shutil.copy(os.path.join(wd, 'RAxML_bestTree.RAxML'), tree) info('Successfully save inferred ML tree to {}.'.format( tree)) except OSError: error('Path of outfile {} is not writeable, saving tree to ' 'file failed.'.format(tree)) sys.exit(1) except OSError: error('Tree inferring failed for {}, executable (exe) {} is ' 'invalid.'.format(msa, exe)) sys.exit(1) finally: shutil.rmtree(wd) return tree
def test_new(self): self.assertEqual('a.name', basename('a.name.new'))
def asr(exe, msa, tree, model, gamma=4, alpha=1.8, freq='', outfile='', verbose=False): """ General use function for (marginal) ancestral states reconstruction (ASR). :param exe: str, path to the executable of an ASR program. :param msa: str, path to the MSA file (must in FASTA format). :param tree: str, path to the tree file (must in NEWICK format) or a NEWICK format tree string (must start with "(" and end with ";"). :param model: str, substitution model for ASR. Either a path to a model file or a valid model string (name of an empirical model plus some other options like gamma category and equilibrium frequency option). If a model file is in use, the file format of the model file depends on the ASR program, see the its documentation for details. :param gamma: int, The number of categories for the discrete gamma rate heterogeneity model. Without setting gamma, RAxML will use CAT model instead, while CODEML will use 4 gamma categories. :param freq: str, the base frequencies of the twenty amino acids. Accept empirical, or estimate, where empirical will set frequencies use the empirical values associated with the specified substitution model, and estimate will use a ML estimate of base frequencies. :param alpha: float, the shape (alpha) for the gamma rate heterogeneity. :param outfile: str, path to the output file. Whiteout setting, results of ancestral states reconstruction will be saved using the filename `[basename].[asrer].tsv`, where basename is the filename of MSA file without known FASTA file extension, asrer is the name of the ASR program (in lower case). The first line of the file will start with '#TREE' and followed by a TAB (\t) and then a NEWICK formatted tree string, the internal nodes were labeled. The second line of the tsv file is intentionally left as a blank line and the rest lines of the file are tab separated sequence IDs and amino acid sequences. :param verbose: bool, invoke verbose or silent (default) process mode. :return: tuple, the paths of the ancestral states file. .. note:: If a tree (with branch lengths and/or internal nodes labeled) is provided, the branch lengths and internal node labels) will be ignored. If the model name combined with Gamma category numbers, i.e. JTT+G4, WAG+G8, etc., only the name of the model will be used. For all models contain G letter, a discrete Gamma model will be used to account for among-site rate variation. If there is a number after letter G, the number will be used to define number of categories in CODEML. For RAxML, the number of categories will always be set to 4 if G presented. """ level = logging.INFO if verbose else logging.ERROR logger.setLevel(level) if os.path.isfile(msa): msa = os.path.abspath(msa) else: error('Ancestral reconstruction aborted, msa {} is not a file or ' 'does not exist.'.format(msa)) sys.exit(1) tree = Tree(tree, leave=True) if not isinstance(model, str): error('Ancestral reconstruction aborted, model {} is not a valid ' 'model name or model file.'.format(model)) sys.exit(1) model = modeling(model) asrer, func = _guess(exe) if not outfile: if msa.endswith('.trimmed.fasta'): name = msa.replace('.trimmed.fasta', '') else: name = msa outfile = '{}.{}.tsv'.format(basename(name), asrer) if os.path.isfile(outfile): info('Found pre-existing ancestral state file.') else: outfile = func(exe, msa, tree, model, gamma, alpha, freq, outfile) return outfile
def test_name(self): self.assertEqual('a.name', basename('a.name'))