def __init__(self): self.plotPrefix = './simulations/simulation.draft.w_refinement_50' self.simCompareFile = './simulations/simCompare.draft.w_refinement_50.full.tsv' self.simCompareMarkerSetOut = './simulations/simCompare.draft.marker_set_table.w_refinement_50.tsv' self.simCompareConditionOut = './simulations/simCompare.draft.condition_table.w_refinement_50.tsv' self.simCompareTaxonomyTableOut = './simulations/simCompare.draft.taxonomy_table.w_refinement_50.tsv' self.simCompareRefinementTableOut = './simulations/simCompare.draft.refinment_table.w_refinement_50.tsv' #self.plotPrefix = './simulations/simulation.scaffolds.draft.w_refinement_50' #self.simCompareFile = './simulations/simCompare.scaffolds.draft.w_refinement_50.full.tsv' #self.simCompareMarkerSetOut = './simulations/simCompare.scaffolds.draft.marker_set_table.w_refinement_50.tsv' #self.simCompareConditionOut = './simulations/simCompare.scaffolds.draft.condition_table.w_refinement_50.tsv' #self.simCompareTaxonomyTableOut = './simulations/simCompare.scaffolds.draft.taxonomy_table.w_refinement_50.tsv' #self.simCompareRefinementTableOut = './simulations/simCompare.scaffolds.draft.refinment_table.w_refinement_50.tsv' #self.plotPrefix = './simulations/simulation.random_scaffolds.w_refinement_50' #self.simCompareFile = './simulations/simCompare.random_scaffolds.w_refinement_50.full.tsv' #self.simCompareMarkerSetOut = './simulations/simCompare.random_scaffolds.marker_set_table.w_refinement_50.tsv' #self.simCompareConditionOut = './simulations/simCompare.random_scaffolds.condition_table.w_refinement_50.tsv' #self.simCompareTaxonomyTableOut = './simulations/simCompare.random_scaffolds.taxonomy_table.w_refinement_50.tsv' #self.simCompareRefinementTableOut = './simulations/simCompare.random_scaffolds.refinment_table.w_refinement_50.tsv' self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') self.compsToConsider = [0.5, 0.7, 0.8, 0.9] #[0.5, 0.7, 0.8, 0.9] self.contsToConsider = [0.05, 0.1, 0.15] #[0.05, 0.1, 0.15] self.dpi = 1200
def __init__(self): self.markerSetBuilder = MarkerSetBuilder() self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') self.contigLens = [1000, 2000, 5000, 10000, 20000, 50000] self.percentComps = [0.5, 0.7, 0.8, 0.9, 0.95, 1.0] self.percentConts = [0.0, 0.05, 0.1, 0.15, 0.2]
def __workerThread(self, ubiquityThreshold, singleCopyThreshold, minGenomes, colocatedDistThreshold, colocatedGenomeThreshold, metadata, queueIn, queueOut): """Process each data item in parallel.""" img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') markerSetBuilder = MarkerSetBuilder() while True: lineage = queueIn.get(block=True, timeout=None) if lineage == None: break if lineage == 'Universal': genomeIds = img.genomeIdsByTaxonomy('prokaryotes', metadata) else: genomeIds = img.genomeIdsByTaxonomy(lineage, metadata) if len(genomeIds) >= minGenomes: markerSet = markerSetBuilder.buildMarkerSet( genomeIds, ubiquityThreshold, singleCopyThreshold, colocatedDistThreshold) colocatedSets = markerSet.markerSet else: colocatedSets = None # allow results to be processed or written to file queueOut.put((lineage, colocatedSets, len(genomeIds)))
def __init__(self): self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') self.colocatedFile = './data/colocated.tsv' self.duplicateSeqs = self.readDuplicateSeqs() self.uniqueIdToLineageStatistics = self.__readNodeMetadata() self.cachedGeneCountTable = None
def __init__(self, outputDir): self.__checkForFastTree() self.derepConcatenatedAlignFile = os.path.join(outputDir, 'genome_tree.concatenated.derep.fasta') self.tree = os.path.join(outputDir, 'genome_tree.final.tre') self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') self.metadata = self.img.genomeMetadata()
def __getUniversalMarkerGenes(self, phyloUbiquityThreshold, phyloSingleCopyThreshold, outputGeneDir): img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') markerSetBuilder = MarkerSetBuilder() metadata = img.genomeMetadata() allTrustedGenomeIds = set() phyloMarkerGenes = {} for lineage in ['Archaea', 'Bacteria']: # get all genomes in lineage print('\nIdentifying all ' + lineage + ' genomes.') trustedGenomeIds = img.genomeIdsByTaxonomy(lineage, metadata) print(' Trusted genomes in lineage: ' + str(len(trustedGenomeIds))) if len(trustedGenomeIds) < 1: print( ' Skipping lineage due to insufficient number of genomes.' ) continue allTrustedGenomeIds.update(trustedGenomeIds) print(' Building marker set.') markerGenes = markerSetBuilder.buildMarkerGenes( trustedGenomeIds, phyloUbiquityThreshold, phyloSingleCopyThreshold) phyloMarkerGenes[lineage] = markerGenes #print lineage #print len(markerGenes) #print 'pfam01379: ', ('pfam01379' in markerGenes) #print '--------------------' # universal marker genes universalMarkerGenes = None for markerGenes in list(phyloMarkerGenes.values()): if universalMarkerGenes == None: universalMarkerGenes = markerGenes else: universalMarkerGenes.intersection_update(markerGenes) fout = open(os.path.join(outputGeneDir, 'phylo_marker_set.txt'), 'w') fout.write(str(universalMarkerGenes)) fout.close() print('') print(' Universal marker genes: ' + str(len(universalMarkerGenes))) return allTrustedGenomeIds, universalMarkerGenes
def run(self, outputDir, ubiquityThreshold, singleCopyThreshold, minGenomes, colocatedDistThreshold, colocatedGenomeThreshold, threads): if not os.path.exists(outputDir): os.makedirs(outputDir) # determine lineages to process img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') metadata = img.genomeMetadata() lineages = img.lineagesSorted(metadata) lineages.append('Universal') # determine HMM model accession numbers pfamIdToPfamAcc = self.__pfamIdToPfamAcc(img) # populate worker queue with data to process workerQueue = mp.Queue() writerQueue = mp.Queue() for lineage in lineages: workerQueue.put(lineage) for _ in range(threads): workerQueue.put(None) workerProc = [ mp.Process(target=self.__workerThread, args=(ubiquityThreshold, singleCopyThreshold, minGenomes, colocatedDistThreshold, colocatedGenomeThreshold, metadata, workerQueue, writerQueue)) for _ in range(threads) ] writeProc = mp.Process( target=self.__writerThread, args=(pfamIdToPfamAcc, ubiquityThreshold, singleCopyThreshold, colocatedDistThreshold, colocatedGenomeThreshold, outputDir, len(lineages), writerQueue)) writeProc.start() for p in workerProc: p.start() for p in workerProc: p.join() writerQueue.put((None, None, None)) writeProc.join()
def __init__(self): self.markerSetBuilder = MarkerSetBuilder() self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv')
def run(self, geneTreeDir, acceptPer, extension, outputDir): # make sure output directory is empty if not os.path.exists(outputDir): os.makedirs(outputDir) files = os.listdir(outputDir) for f in files: os.remove(os.path.join(outputDir, f)) img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') metadata = img.genomeMetadata() files = os.listdir(geneTreeDir) print('Identifying gene trees with only conspecific paralogous genes:') filteredGeneTrees = 0 retainedGeneTrees = 0 for f in files: if not f.endswith(extension): continue geneId = f[0:f.find('.')] print(' Testing gene tree: ' + geneId) tree = dendropy.Tree.get_from_path(os.path.join(geneTreeDir, f), schema='newick', as_rooted=False, preserve_underscores=True) taxa = tree.leaf_nodes() numTaxa = len(taxa) print(' Genes in tree: ' + str(numTaxa)) # root tree with archaeal genomes rerootTree = RerootTree() rerootTree.reroot(tree) # get species name of each taxa leafNodeToSpeciesName = {} for t in taxa: genomeId = t.taxon.label.split('|')[0] genus = metadata[genomeId]['taxonomy'][5] sp = metadata[genomeId]['taxonomy'][6].lower() leafNodeToSpeciesName[t.taxon.label] = genus + ' ' + sp # find all paralogous genes print(' Finding paralogous genes.') paralogs = defaultdict(set) for i in range(0, len(taxa)): genomeId = taxa[i].taxon.label.split('|')[0] for j in range(i + 1, len(taxa)): # genes from the same genome are paralogs, but we filter out # those that are identical (distance of 0 on the tree) to # speed up computation and because these clearly do not # adversely effect phylogenetic inference if genomeId == taxa[j].taxon.label.split( '|')[0] and self.__patristicDist( tree, taxa[i], taxa[j]) > 0: paralogs[genomeId].add(taxa[i].taxon.label) paralogs[genomeId].add(taxa[j].taxon.label) print(' Paralogous genes: ' + str(len(paralogs))) # check if paralogous genes are conspecific print(' Determining if paralogous genes are conspecific.') nonConspecificGenomes = [] for genomeId, taxaLabels in paralogs.iteritems(): lcaNode = tree.mrca(taxon_labels=taxaLabels) children = lcaNode.leaf_nodes() species = set() for child in children: childGenomeId = child.taxon.label.split('|')[0] genus = metadata[childGenomeId]['taxonomy'][5] sp = metadata[childGenomeId]['taxonomy'][6].lower() if sp != '' and sp != 'unclassified' and genus != 'unclassified': species.add(genus + ' ' + sp) if len(species) > 1: nonConspecificGenomes.append(genomeId) if len(nonConspecificGenomes) > acceptPer * numTaxa: filteredGeneTrees += 1 print(' Tree is not conspecific for the following genome: ' + str(nonConspecificGenomes)) else: retainedGeneTrees += 1 if len(nonConspecificGenomes) > 1: print( ' An acceptable number of genomes are not conspecific: ' + str(nonConspecificGenomes)) else: print(' Tree is conspecific.') os.system('cp ' + os.path.join(geneTreeDir, f) + ' ' + os.path.join(outputDir, f)) print('') print('Filtered gene trees: ' + str(filteredGeneTrees)) print('Retained gene trees: ' + str(retainedGeneTrees))
def __init__(self): img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') self.metadata = img.genomeMetadata()
def __init__(self): self.img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv')
def run(self, geneTreeDir, treeExtension, consistencyThreshold, minTaxaForAverage, outputFile, outputDir): # make sure output directory is empty if not os.path.exists(outputDir): os.makedirs(outputDir) files = os.listdir(outputDir) for f in files: if os.path.isfile(os.path.join(outputDir, f)): os.remove(os.path.join(outputDir, f)) # get TIGRFam info descDict = {} files = os.listdir('/srv/db/tigrfam/13.0/TIGRFAMs_13.0_INFO') for f in files: shortDesc = longDesc = '' for line in open('/srv/db/tigrfam/13.0/TIGRFAMs_13.0_INFO/' + f): lineSplit = line.split(' ') if lineSplit[0] == 'AC': acc = lineSplit[1].strip() elif lineSplit[0] == 'DE': shortDesc = lineSplit[1].strip() elif lineSplit[0] == 'CC': longDesc = lineSplit[1].strip() descDict[acc] = [shortDesc, longDesc] # get PFam info for line in open('/srv/db/pfam/27/Pfam-A.clans.tsv'): lineSplit = line.split('\t') acc = lineSplit[0] shortDesc = lineSplit[3] longDesc = lineSplit[4].strip() descDict[acc] = [shortDesc, longDesc] # get IMG taxonomy img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') metadata = img.genomeMetadata() genomeIdToTaxonomy = {} for genomeId, m in metadata.iteritems(): genomeIdToTaxonomy[genomeId] = m['taxonomy'] # perform analysis for each tree treeFiles = os.listdir(geneTreeDir) allResults = {} allTaxa = [set([]), set([]), set([])] taxaCounts = {} avgConsistency = {} for treeFile in treeFiles: if not treeFile.endswith(treeExtension): continue print treeFile tree = dendropy.Tree.get_from_path(os.path.join( geneTreeDir, treeFile), schema='newick', as_rooted=True, preserve_underscores=True) domainConsistency = {} phylaConsistency = {} classConsistency = {} consistencyDict = [ domainConsistency, phylaConsistency, classConsistency ] # get abundance of taxa at different taxonomic ranks totals = [{}, {}, {}] leaves = tree.leaf_nodes() print ' Number of leaves: ' + str(len(leaves)) totalValidLeaves = 0 for leaf in leaves: genomeId = self.__genomeId(leaf.taxon.label) if genomeId not in metadata: print '[Error] Genome is missing metadata: ' + genomeId sys.exit() totalValidLeaves += 1 taxonomy = genomeIdToTaxonomy[genomeId] for r in xrange(0, 3): totals[r][taxonomy[r]] = totals[r].get(taxonomy[r], 0) + 1 consistencyDict[r][taxonomy[r]] = 0 allTaxa[r].add(taxonomy[r]) taxaCounts[treeFile] = [ totalValidLeaves, totals[0].get('Bacteria', 0), totals[0].get('Archaea', 0) ] # find highest consistency nodes (congruent descendant taxa / (total taxa + incongruent descendant taxa)) internalNodes = tree.internal_nodes() for node in internalNodes: leaves = node.leaf_nodes() for r in xrange(0, 3): leafCounts = {} for leaf in leaves: genomeId = self.__genomeId(leaf.taxon.label) taxonomy = genomeIdToTaxonomy[genomeId] leafCounts[taxonomy[r]] = leafCounts.get( taxonomy[r], 0) + 1 # calculate consistency for node for taxa in consistencyDict[r]: totalTaxaCount = totals[r][taxa] if totalTaxaCount <= 1 or taxa == 'unclassified': consistencyDict[r][taxa] = 'N/A' continue taxaCount = leafCounts.get(taxa, 0) incongruentTaxa = len(leaves) - taxaCount c = float(taxaCount) / (totalTaxaCount + incongruentTaxa) if c > consistencyDict[r][taxa]: consistencyDict[r][taxa] = c # consider clan in other direction since the trees are unrooted taxaCount = totalTaxaCount - leafCounts.get(taxa, 0) incongruentTaxa = totalValidLeaves - len( leaves) - taxaCount c = float(taxaCount) / (totalTaxaCount + incongruentTaxa) if c > consistencyDict[r][taxa]: consistencyDict[r][taxa] = c # write results consistencyDir = os.path.join(outputDir, 'consistency') if not os.path.exists(consistencyDir): os.makedirs(consistencyDir) fout = open( os.path.join(consistencyDir, treeFile + '.results.tsv'), 'w') fout.write('Tree') for r in xrange(0, 3): for taxa in sorted(consistencyDict[r].keys()): fout.write('\t' + taxa) fout.write('\n') fout.write(treeFile) for r in xrange(0, 3): for taxa in sorted(consistencyDict[r].keys()): if consistencyDict[r][taxa] != 'N/A': fout.write('\t%.2f' % (consistencyDict[r][taxa] * 100)) else: fout.write('\tN/A') fout.close() # calculate average consistency at each taxonomic rank average = [] for r in xrange(0, 3): sumConsistency = [] for taxa in consistencyDict[r]: if totals[r][taxa] > minTaxaForAverage and consistencyDict[ r][taxa] != 'N/A': sumConsistency.append(consistencyDict[r][taxa]) if len(sumConsistency) > 0: average.append(sum(sumConsistency) / len(sumConsistency)) else: average.append(0) avgConsistency[treeFile] = average allResults[treeFile] = consistencyDict print ' Average consistency: ' + str( average) + ', mean = %.2f' % (sum(average) / len(average)) print '' # print out combined results fout = open(outputFile, 'w') fout.write( 'Tree\tShort Desc.\tLong Desc.\tAlignment Length\t# Taxa\t# Bacteria\t# Archaea\tAvg. Consistency\tAvg. Domain Consistency\tAvg. Phylum Consistency\tAvg. Class Consistency' ) for r in xrange(0, 3): for t in sorted(allTaxa[r]): fout.write('\t' + t) fout.write('\n') filteredGeneTrees = 0 retainedGeneTrees = 0 for treeFile in sorted(allResults.keys()): consistencyDict = allResults[treeFile] treeId = treeFile[0:treeFile.find('.')].replace('pfam', 'PF') fout.write(treeId + '\t' + descDict[treeId][0] + '\t' + descDict[treeId][1]) # Taxa count fout.write('\t' + str(taxaCounts[treeFile][0]) + '\t' + str(taxaCounts[treeFile][1]) + '\t' + str(taxaCounts[treeFile][2])) avgCon = 0 for r in xrange(0, 3): avgCon += avgConsistency[treeFile][r] avgCon /= 3 fout.write('\t' + str(avgCon)) if avgCon >= consistencyThreshold: retainedGeneTrees += 1 os.system('cp ' + os.path.join(geneTreeDir, treeFile) + ' ' + os.path.join(outputDir, treeFile)) else: filteredGeneTrees += 1 print 'Filtered % s with an average consistency of %.4f.' % ( treeFile, avgCon) for r in xrange(0, 3): fout.write('\t' + str(avgConsistency[treeFile][r])) for r in xrange(0, 3): for t in sorted(allTaxa[r]): if t in consistencyDict[r]: if consistencyDict[r][t] != 'N/A': fout.write('\t%.2f' % (consistencyDict[r][t] * 100)) else: fout.write('\tN/A') else: fout.write('\tN/A') fout.write('\n') fout.close() print 'Retained gene trees: ' + str(retainedGeneTrees) print 'Filtered gene trees: ' + str(filteredGeneTrees)
def run(self, geneTreeDir, alignmentDir, extension, outputAlignFile, outputTree, outputTaxonomy, bSupportValues=False): # read gene trees print 'Reading gene trees.' geneIds = set() files = os.listdir(geneTreeDir) for f in files: if f.endswith('.tre'): geneId = f[0:f.find('.')] geneIds.add(geneId) # write out genome tree taxonomy print 'Reading trusted genomes.' img = IMG('/srv/whitlam/bio/db/checkm/img/img_metadata.tsv', '/srv/whitlam/bio/db/checkm/pfam/tigrfam2pfam.tsv') genomeIds = img.genomeMetadata().keys() self.__taxonomy(img, genomeIds, outputTaxonomy) print ' There are %d trusted genomes.' % (len(genomeIds)) # get genes in genomes print 'Reading all PFAM and TIGRFAM hits in trusted genomes.' genesInGenomes = self.__genesInGenomes(genomeIds) # read alignment files print 'Reading alignment files.' alignments = {} genomeIds = set() files = os.listdir(alignmentDir) for f in files: geneId = f[0:f.find('.')] if f.endswith(extension) and geneId in geneIds: seqs = readFasta(os.path.join(alignmentDir, f)) imgGeneId = geneId if imgGeneId.startswith('PF'): imgGeneId = imgGeneId.replace('PF', 'pfam') seqs = self.__filterParalogs(seqs, imgGeneId, genesInGenomes) genomeIds.update(set(seqs.keys())) alignments[geneId] = seqs # create concatenated alignment print 'Concatenating alignments:' concatenatedSeqs = {} totalAlignLen = 0 for geneId in sorted(alignments.keys()): seqs = alignments[geneId] alignLen = len(seqs[seqs.keys()[0]]) print ' ' + str(geneId) + ',' + str(alignLen) totalAlignLen += alignLen for genomeId in genomeIds: if genomeId in seqs: # append alignment concatenatedSeqs['IMG_' + genomeId] = concatenatedSeqs.get( 'IMG_' + genomeId, '') + seqs[genomeId] else: # missing gene concatenatedSeqs['IMG_' + genomeId] = concatenatedSeqs.get( 'IMG_' + genomeId, '') + '-' * alignLen print ' Total alignment length: ' + str(totalAlignLen) # save concatenated alignment writeFasta(concatenatedSeqs, outputAlignFile) # infer genome tree print 'Inferring genome tree.' outputLog = outputTree[0:outputTree.rfind('.')] + '.log' supportStr = ' ' if not bSupportValues: supportStr = ' -nosupport ' cmd = 'FastTreeMP' + supportStr + '-wag -gamma -log ' + outputLog + ' ' + outputAlignFile + ' > ' + outputTree os.system(cmd)