def create_crosstable(sico_files, target_crosstable): """Create crosstable with vertically the orthologs, horizontally the genomes, and gene IDs at intersections.""" with open(target_crosstable, mode='w') as write_handle: # Create dictionaries mapping genomes to gene IDs per sico file row_data = [(sico_file, dict(itemgetter(0, 2)(fasta_record.id.split('|')) for fasta_record in SeqIO.parse(sico_file, 'fasta'))) for sico_file in sico_files] # Retrieve unique genomes across all sico files, just to be safe genomes = sorted(set(key for row in row_data for key in row[1].keys())) genome_dicts = select_genomes_by_ids(genomes).values() # Write out values to file write_handle.write('\t' + '\t'.join(genomes)) write_handle.write('\tCOGs\tProduct\n') for sico_file, row in row_data: ortholog = os.path.split(sico_file)[1].split('.')[0] write_handle.write(ortholog + '\t') write_handle.write('\t'.join(row.get(genome, '') for genome in genomes)) # Parse sequence records again, but now to retrieve cogs and products seq_records = list(SeqIO.parse(sico_file, 'fasta')) # COGs cogs = find_cogs_in_sequence_records(seq_records) write_handle.write('\t' + ','.join(cogs)) # Product product = get_most_recent_gene_name(genome_dicts, seq_records) write_handle.write('\t' + product) # New line write_handle.write('\n')
def main(args): """Main function called when run from command line or as part of pipeline.""" usage = """ Usage: concatenate_orthologs.py --orthologs-zip=FILE archive of orthologous genes in FASTA format --coding-regions=FILE destination file path archive of trimmed orthologous coding regions per genomes --concatemer=FILE destination file path for super-concatemer of all genomes --taxon-a=FILE destination file path for genome IDs for taxon A --taxon-b=FILE destination file path for genome IDs for taxon B --tree=FILE destination file path for tree visualization """ options = ['orthologs-zip', 'coding-regions', 'concatemer', 'taxon-a', 'taxon-b', 'tree'] orthologs_zip, target_coding_regions, target_concat_file, target_taxon_a, target_taxon_b, target_tree = \ parse_options(usage, options, args) # Run filtering in a temporary folder, to prevent interference from simultaneous runs run_dir = tempfile.mkdtemp(prefix='concatemer_tree_') # Extract files from zip archive temp_dir = create_directory('orthologs', inside_dir=run_dir) ortholog_files = extract_archive_of_files(orthologs_zip, temp_dir) # Separate out orthologs per genome to create trimmed coding region files per genome genome_coding_regions_files = coding_regions_per_genome(run_dir, ortholog_files) create_archive_of_files(target_coding_regions, genome_coding_regions_files) # Concatenate coding region files per genome concatemer_files = concatemer_per_genome(run_dir, genome_coding_regions_files) # Create super concatemer create_super_concatemer(concatemer_files, target_concat_file) # Determine the taxa present in the super concatemer tree by building a phylogenetic tree from genome concatemer and # reading genome ids in the two largest clades. super_distance_file = _run_dna_dist(run_dir, target_concat_file) super_tree_file = _run_neighbor(run_dir, super_distance_file) genome_ids_a, genome_ids_b = _read_taxa_from_tree(super_tree_file) # Map Project IDs to Organism names id_to_name_map = dict((gid, genome['Organism/Name']) for gid, genome in select_genomes_by_ids(genome_ids_a + genome_ids_b).iteritems()) # Write Project IDs and Organism Names to files, with a fallback to genome_id for external genome with open(target_taxon_a, mode='w') as write_handle: for genome_id in genome_ids_a: write_handle.write('{id}\t{name}\n'.format(id=genome_id, name=id_to_name_map.get(genome_id, genome_id))) with open(target_taxon_b, mode='w') as write_handle: for genome_id in genome_ids_b: write_handle.write('{id}\t{name}\n'.format(id=genome_id, name=id_to_name_map.get(genome_id, genome_id))) # Visualize tree visualize_tree(super_tree_file, id_to_name_map, target_tree) # Remove unused files to free disk space shutil.rmtree(run_dir) # Exit after a comforting log message log.info('Produced: \n%s\n%s\n%s\n%s\n%s', target_coding_regions, target_concat_file, target_taxon_a, target_taxon_b, target_tree)
def main(args): """Main function called when run from command line or as part of pipeline.""" usage = """ Usage: translate.py --genomes=FILE file with selected genome IDs followed by Organism Name on each line --external-zip=FILE optional archive of user provided external genomes containing formatted nucleotide fasta files --dna-zip=FILE destination file path for zip archive of extracted DNA files --protein-zip=FILE destination file path for zip archive of translated protein files """ options = ['genomes', 'external-zip=?', 'dna-zip', 'protein-zip'] genome_ids_file, external_zip, dna_zipfile, protein_zipfile = parse_options(usage, options, args) dna_files = [] protein_files = [] # Read GenBank Project IDs from genomes_file, each on their own line with open(genome_ids_file) as read_handle: genome_ids = [line.split()[0] for line in read_handle if not line.startswith('#') and 'external genome' not in line] if len(genome_ids): # Retrieve associated genome dictionaries from complete genomes table genomes = select_genomes_by_ids(genome_ids).values() genomes = sorted(genomes, key=itemgetter('Organism/Name')) # Actually translate the genomes to produced a set of files for both dna files & protein files dna_files, protein_files = translate_genomes(genomes) # Also translate the external genomes if external_zip: # Extract external genomes archive external_dir = tempfile.mkdtemp(prefix='external_genomes_') external_dna_files = extract_archive_of_files(external_zip, external_dir) # Append IDs of external fasta files to genome IDs file _append_external_genomes(external_dna_files, genome_ids_file) # Translate individual files external_protein_files = [translate_fasta_coding_regions(dna_file) for dna_file in external_dna_files] # Add the files to the appropriate collections dna_files.extend(external_dna_files) protein_files.extend(external_protein_files) # Write the produced files to command line argument filenames create_archive_of_files(dna_zipfile, dna_files) create_archive_of_files(protein_zipfile, protein_files) # Do not clean up extracted DNA files or Protein translations: Keep them as cache # But do clean up external_dir now that the compressed archives are created if external_zip: shutil.rmtree(external_dir) # Exit after a comforting log message log.info("Produced: \n%s &\n%s", dna_zipfile, protein_zipfile)
def test_translate_93125_2(self): # Select genomes genomes = select_genomes_by_ids(['93125.2']).values() # Call translate aafiles = translate.translate_genomes(genomes)[1] # Verify no header appears twice headers = [record.id for record in SeqIO.parse(aafiles[0], 'fasta')] self.assertEqual(len(headers), len(set(headers)))
def _table_calculations(genome_ids_a, genome_ids_b, sico_files, phipack_values): '''Perform calculations for comparsion of genome_ids_a with genome_ids_b.''' # retrieve genomes once for both genomes_a = select_genomes_by_ids(genome_ids_a).values() # dictionary to hold the values calculated per file calculations = [] # loop over orthologs for sico_file in sico_files: # parse alignment alignment = AlignIO.read(sico_file, 'fasta') # split alignments alignment_a = MultipleSeqAlignment(seqr for seqr in alignment if seqr.id.split('|')[0] in genome_ids_a) alignment_b = MultipleSeqAlignment(seqr for seqr in alignment if seqr.id.split('|')[0] in genome_ids_b) # calculate codeml values codeml_values = _get_codeml_values(alignment_a, alignment_b) # create gathering instance of clade_calcs instance = clade_calcs(alignment_a, genomes_a) # store ortholog name retrieved from filename ortholog = os.path.basename(sico_file).split('.')[0] instance.values[ORTHOLOG] = ortholog # add codeml_values to clade_calcs instance values instance.values.update(codeml_values) # add phipack values for this file instance.values.update(phipack_values[sico_file]) # add COG digits and letters _extract_cog_digits_and_letters(instance) # add SFS related values _codon_site_freq_spec(instance) # add additional deduced calculation _add_combined_calculations(instance) # store the clade_calc values calculations.append(instance) # calculcate mean and averages max_nton = len(genome_ids_a) // 2 sum_stats, mean_stats = _calculcate_mean_and_averages(calculations, max_nton) # neutrality index calculation and bootstrapping ni_stats, ni_lower_stats, ni_upper_stats = _neutrality_indices(calculations) # finally append statistics to calculations so they show up in file calculations.extend((sum_stats, mean_stats, ni_stats, ni_lower_stats, ni_upper_stats)) return calculations
def _occurences_and_cogs(genome_ids, ortholog_files): """Generator that returns how many sequences exist per genome in each ortholog in order and which COGs occur.""" genomes = select_genomes_by_ids(genome_ids).values() for fasta_file in ortholog_files: records = tuple(SeqIO.parse(fasta_file, 'fasta')) ids = [record.id.split('|')[0] for record in records] count_per_id = [ids.count(genome_id) for genome_id in genome_ids] cogs = sorted(find_cogs_in_sequence_records(records)) ortholog_nr = os.path.splitext(os.path.split(fasta_file)[1])[0] for record in records: # SeqIO mucks up ids containing spaces, so we have to assign description as value for id record.id = record.description product = get_most_recent_gene_name(genomes, records) yield count_per_id, ortholog_nr, cogs, product
def _occurences_and_cogs(genome_ids, ortholog_files): """Generator that returns how many sequences exist per genome in each ortholog in order and which COGs occur.""" genomes = select_genomes_by_ids(genome_ids).values() for fasta_file in ortholog_files: records = tuple(SeqIO.parse(fasta_file, 'fasta')) ids = [record.id.split('|')[0] for record in records] count_per_id = [ids.count(genome_id) for genome_id in genome_ids] cogs = sorted(find_cogs_in_sequence_records(records)) ortholog_nr = os.path.splitext(os.path.split(fasta_file)[1])[0] for record in records: # SeqIO mucks up ids containing spaces, so we have to assign description as value for id record.id = record.description product = get_most_recent_gene_name(genomes, records) yield count_per_id, ortholog_nr, cogs, product
def test_translate_genomes(self): # Select genomes genomes = select_genomes_by_ids(['13305.1']).values() # Call translate dnafiles, aafiles = translate.translate_genomes(genomes) # Verify expected output first_header = '13305.1|NC_008253.1|YP_667942.1|None|thr' first = next(SeqIO.parse(dnafiles[0], 'fasta')) self.assertEqual(first_header, first.id) first = next(SeqIO.parse(aafiles[0], 'fasta')) self.assertEqual(first_header, first.id) # Verify no header appears twice headers = [record.id for record in SeqIO.parse(aafiles[0], 'fasta')] self.assertEqual(len(headers), len(set(headers)))
def _phipack_for_all_orthologs(run_dir, aligned_files, stats_file): """Filter aligned fasta files where there is evidence of recombination when inspecting PhiPack values. Return two collections of aligned files, the first without recombination, the second with recombination.""" log.info('Running PhiPack for %i orthologs to find recombination', len(aligned_files)) # Create separate directory for phipack related values phipack_dir = create_directory('phipack', inside_dir=run_dir) with open(stats_file, mode='w') as write_handle: write_handle.write('\t'.join([ 'Ortholog', 'Informative sites', 'Phi', 'Max Chi^2', 'NSS', 'COGs', 'Product' ]) + '\n') # Retrieve unique genomes from first ortholog file genome_ids = set( fasta_record.id.split('|')[0] for fasta_record in SeqIO.parse(aligned_files[0], 'fasta')) genome_dicts = select_genomes_by_ids(genome_ids).values() # Assign ortholog files to the correct collection based on whether they show recombination for ortholog_file in aligned_files: orth_name = os.path.split(ortholog_file)[1].split('.')[0] # Parse tree file to ensure all genome_ids_a & genome_ids_b group together in the tree phipack_values = run_phipack(phipack_dir, ortholog_file) # Write PhiPack values to line write_handle.write( '{0}\t{1[PhiPack sites]}\t{1[Phi]}\t{1[Max Chi^2]}\t{1[NSS]}'. format(orth_name, phipack_values)) # Parse sequence records again, but now to retrieve cogs and products seq_records = list(SeqIO.parse(ortholog_file, 'fasta')) # COGs cogs = find_cogs_in_sequence_records(seq_records) write_handle.write('\t' + ','.join(cogs)) # Product product = get_most_recent_gene_name(genome_dicts, seq_records) write_handle.write('\t' + product) # End line write_handle.write('\n')
def _phipack_for_all_orthologs(run_dir, aligned_files, stats_file): """Filter aligned fasta files where there is evidence of recombination when inspecting PhiPack values. Return two collections of aligned files, the first without recombination, the second with recombination.""" log.info('Running PhiPack for %i orthologs to find recombination', len(aligned_files)) # Create separate directory for phipack related values phipack_dir = create_directory('phipack', inside_dir=run_dir) with open(stats_file, mode='w') as write_handle: write_handle.write('\t'.join(['Ortholog', 'Informative sites', 'Phi', 'Max Chi^2', 'NSS', 'COGs', 'Product']) + '\n') # Retrieve unique genomes from first ortholog file genome_ids = set(fasta_record.id.split('|')[0] for fasta_record in SeqIO.parse(aligned_files[0], 'fasta')) genome_dicts = select_genomes_by_ids(genome_ids).values() # Assign ortholog files to the correct collection based on whether they show recombination for ortholog_file in aligned_files: orth_name = os.path.split(ortholog_file)[1].split('.')[0] # Parse tree file to ensure all genome_ids_a & genome_ids_b group together in the tree phipack_values = run_phipack(phipack_dir, ortholog_file) # Write PhiPack values to line write_handle.write('{0}\t{1[PhiPack sites]}\t{1[Phi]}\t{1[Max Chi^2]}\t{1[NSS]}'.format(orth_name, phipack_values)) # Parse sequence records again, but now to retrieve cogs and products seq_records = list(SeqIO.parse(ortholog_file, 'fasta')) # COGs cogs = find_cogs_in_sequence_records(seq_records) write_handle.write('\t' + ','.join(cogs)) # Product product = get_most_recent_gene_name(genome_dicts, seq_records) write_handle.write('\t' + product) # End line write_handle.write('\n')
def _table_calculations(genome_ids_a, genome_ids_b, sico_files, phipack_values): '''Perform calculations for comparsion of genome_ids_a with genome_ids_b.''' # retrieve genomes once for both genomes_a = select_genomes_by_ids(genome_ids_a).values() # dictionary to hold the values calculated per file calculations = [] # loop over orthologs for sico_file in sico_files: # parse alignment alignment = AlignIO.read(sico_file, 'fasta') # split alignments alignment_a = MultipleSeqAlignment( seqr for seqr in alignment if seqr.id.split('|')[0] in genome_ids_a) alignment_b = MultipleSeqAlignment( seqr for seqr in alignment if seqr.id.split('|')[0] in genome_ids_b) # calculate codeml values codeml_values = _get_codeml_values(alignment_a, alignment_b) # create gathering instance of clade_calcs instance = clade_calcs(alignment_a, genomes_a) # store ortholog name retrieved from filename ortholog = os.path.basename(sico_file).split('.')[0] instance.values[ORTHOLOG] = ortholog # add codeml_values to clade_calcs instance values instance.values.update(codeml_values) # add phipack values for this file instance.values.update(phipack_values[sico_file]) # add COG digits and letters _extract_cog_digits_and_letters(instance) # add SFS related values _codon_site_freq_spec(instance) # add additional deduced calculation _add_combined_calculations(instance) # store the clade_calc values calculations.append(instance) # calculcate mean and averages max_nton = len(genome_ids_a) // 2 sum_stats, mean_stats = _calculcate_mean_and_averages( calculations, max_nton) # neutrality index calculation and bootstrapping ni_stats, ni_lower_stats, ni_upper_stats = _neutrality_indices( calculations) # finally append statistics to calculations so they show up in file calculations.extend( (sum_stats, mean_stats, ni_stats, ni_lower_stats, ni_upper_stats)) return calculations
def main(args): """Main function called when run from command line or as part of pipeline.""" usage = """ Usage: concatenate_orthologs.py --orthologs-zip=FILE archive of orthologous genes in FASTA format --coding-regions=FILE destination file path archive of trimmed orthologous coding regions per genomes --concatemer=FILE destination file path for super-concatemer of all genomes --taxon-a=FILE destination file path for genome IDs for taxon A --taxon-b=FILE destination file path for genome IDs for taxon B --tree=FILE destination file path for tree visualization """ options = [ 'orthologs-zip', 'coding-regions', 'concatemer', 'taxon-a', 'taxon-b', 'tree' ] orthologs_zip, target_coding_regions, target_concat_file, target_taxon_a, target_taxon_b, target_tree = \ parse_options(usage, options, args) # Run filtering in a temporary folder, to prevent interference from simultaneous runs run_dir = tempfile.mkdtemp(prefix='concatemer_tree_') # Extract files from zip archive temp_dir = create_directory('orthologs', inside_dir=run_dir) ortholog_files = extract_archive_of_files(orthologs_zip, temp_dir) # Separate out orthologs per genome to create trimmed coding region files per genome genome_coding_regions_files = coding_regions_per_genome( run_dir, ortholog_files) create_archive_of_files(target_coding_regions, genome_coding_regions_files) # Concatenate coding region files per genome concatemer_files = concatemer_per_genome(run_dir, genome_coding_regions_files) # Create super concatemer create_super_concatemer(concatemer_files, target_concat_file) # Determine the taxa present in the super concatemer tree by building a phylogenetic tree from genome concatemer and # reading genome ids in the two largest clades. super_distance_file = _run_dna_dist(run_dir, target_concat_file) super_tree_file = _run_neighbor(run_dir, super_distance_file) genome_ids_a, genome_ids_b = _read_taxa_from_tree(super_tree_file) # Map Project IDs to Organism names id_to_name_map = dict( (gid, genome['Organism/Name']) for gid, genome in select_genomes_by_ids(genome_ids_a + genome_ids_b).iteritems()) # Write Project IDs and Organism Names to files, with a fallback to genome_id for external genome with open(target_taxon_a, mode='w') as write_handle: for genome_id in genome_ids_a: write_handle.write('{id}\t{name}\n'.format(id=genome_id, name=id_to_name_map.get( genome_id, genome_id))) with open(target_taxon_b, mode='w') as write_handle: for genome_id in genome_ids_b: write_handle.write('{id}\t{name}\n'.format(id=genome_id, name=id_to_name_map.get( genome_id, genome_id))) # Visualize tree visualize_tree(super_tree_file, id_to_name_map, target_tree) # Remove unused files to free disk space shutil.rmtree(run_dir) # Exit after a comforting log message log.info('Produced: \n%s\n%s\n%s\n%s\n%s', target_coding_regions, target_concat_file, target_taxon_a, target_taxon_b, target_tree)