def run(self, ubiquityThreshold, singleCopyThreshold, minGenomes, mostSpecificRank, minMarkers, completenessThreshold, contaminationThreshold):
        print 'Ubiquity threshold: ' + str(ubiquityThreshold)
        print 'Single-copy threshold: ' + str(singleCopyThreshold)
        print 'Min. genomes: ' + str(minGenomes)
        print 'Most specific taxonomic rank: ' + str(mostSpecificRank)
        print 'Min markers: ' + str(minMarkers)
        print 'Completeness threshold: ' + str(completenessThreshold)
        print 'Contamination threshold: ' + str(contaminationThreshold)

        img = IMG()
        markerset = MarkerSet()

        lineages = img.lineagesByCriteria(minGenomes, mostSpecificRank)

        degenerateGenomes = {}
        for lineage in lineages:
            genomeIds = img.genomeIdsByTaxonomy(lineage, 'Final')

            print ''
            print 'Lineage ' + lineage + ' contains ' + str(len(genomeIds)) + ' genomes.'

            # get table of PFAMs and do some initial filtering to remove PFAMs that are
            # clearly not going to pass the ubiquity and single-copy thresholds
            countTable = img.countTable(genomeIds)
            countTable = img.filterTable(genomeIds, countTable, ubiquityThreshold*0.9, singleCopyThreshold*0.9)

            markerGenes = markerset.markerGenes(genomeIds, countTable, ubiquityThreshold*len(genomeIds), singleCopyThreshold*len(genomeIds))
            if len(markerGenes) < minMarkers:
                continue

            geneDistTable = img.geneDistTable(genomeIds, markerGenes, spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable)
            colocatedSets = markerset.colocatedSets(colocatedGenes, markerGenes)

            for genomeId in genomeIds:
                completeness, contamination = markerset.genomeCheck(colocatedSets, genomeId, countTable)

                if completeness < completenessThreshold or contamination > contaminationThreshold:
                    degenerateGenomes[genomeId] = degenerateGenomes.get(genomeId, []) + [[lineage.split(';')[-1].strip(), len(genomeIds), len(colocatedSets), completeness, contamination]]

        # write out degenerate genomes
        metadata = img.genomeMetadata('Final')

        fout = open('./data/degenerate_genomes.tsv', 'w')
        fout.write('Genome Id\tTaxonomy\tGenome Size (Gbps)\tScaffolds\tBiotic Relationships\tStatus\tLineage\t# genomes\tMarker set size\tCompleteness\tContamination\n')
        for genomeId, data in degenerateGenomes.iteritems():
            fout.write(genomeId + '\t' + '; '.join(metadata[genomeId]['taxonomy']) + '\t%.2f' % (float(metadata[genomeId]['genome size']) / 1e6) + '\t' + str(metadata[genomeId]['scaffold count']))
            fout.write('\t' + metadata[genomeId]['biotic relationships'] + '\t' + metadata[genomeId]['status'])

            for d in data:
                fout.write('\t' + d[0] + '\t' + str(d[1]) + '\t' + str(d[2]) + '\t%.3f\t%.3f' % (d[3], d[4]))
            fout.write('\n')

        fout.close()
Example #2
0
    def run(
        self,
        ubiquityThreshold,
        singleCopyThreshold,
        minGenomes,
        minMarkers,
        mostSpecificRank,
        distThreshold,
        genomeThreshold,
    ):
        img = IMG()
        markerset = MarkerSet()

        lineages = img.lineagesSorted(mostSpecificRank)

        fout = open("./data/colocated.tsv", "w", 1)
        fout.write("Lineage\t# genomes\t# markers\t# co-located sets\tCo-located markers\n")

        lineageCount = 0
        for lineage in lineages:
            lineageCount += 1

            genomeIds = img.genomeIdsByTaxonomy(lineage, "Final")
            if len(genomeIds) < minGenomes:
                continue

            countTable = img.countTable(genomeIds)
            markerGenes = markerset.markerGenes(
                genomeIds, countTable, ubiquityThreshold * len(genomeIds), singleCopyThreshold * len(genomeIds)
            )

            geneDistTable = img.geneDistTable(genomeIds, markerGenes)
            colocatedGenes = markerset.colocatedGenes(geneDistTable, distThreshold, genomeThreshold)
            colocatedSets = markerset.colocatedSets(colocatedGenes, markerGenes)
            if len(colocatedSets) < minMarkers:
                continue

            print "\nLineage " + lineage + " contains " + str(len(genomeIds)) + " genomes (" + str(
                lineageCount
            ) + " of " + str(len(lineages)) + ")."
            print "  Marker genes: " + str(len(markerGenes))
            print "  Co-located gene sets: " + str(len(colocatedSets))

            fout.write(
                lineage + "\t" + str(len(genomeIds)) + "\t" + str(len(markerGenes)) + "\t" + str(len(colocatedSets))
            )
            for cs in colocatedSets:
                fout.write("\t" + ", ".join(cs))
            fout.write("\n")

        fout.close()
Example #3
0
    def run(self, ubiquityThreshold, singleCopyThreshold, minGenomes,
            minMarkers, mostSpecificRank, distThreshold, genomeThreshold):
        img = IMG()
        markerset = MarkerSet()

        lineages = img.lineagesSorted(mostSpecificRank)

        fout = open('./data/colocated.tsv', 'w', 1)
        fout.write(
            'Lineage\t# genomes\t# markers\t# co-located sets\tCo-located markers\n'
        )

        lineageCount = 0
        for lineage in lineages:
            lineageCount += 1

            genomeIds = img.genomeIdsByTaxonomy(lineage, 'Final')
            if len(genomeIds) < minGenomes:
                continue

            countTable = img.countTable(genomeIds)
            markerGenes = markerset.markerGenes(
                genomeIds, countTable, ubiquityThreshold * len(genomeIds),
                singleCopyThreshold * len(genomeIds))

            geneDistTable = img.geneDistTable(genomeIds,
                                              markerGenes,
                                              spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable,
                                                      distThreshold,
                                                      genomeThreshold)
            colocatedSets = markerset.colocatedSets(colocatedGenes,
                                                    markerGenes)
            if len(colocatedSets) < minMarkers:
                continue

            print '\nLineage ' + lineage + ' contains ' + str(len(
                genomeIds)) + ' genomes (' + str(lineageCount) + ' of ' + str(
                    len(lineages)) + ').'
            print '  Marker genes: ' + str(len(markerGenes))
            print '  Co-located gene sets: ' + str(len(colocatedSets))

            fout.write(lineage + '\t' + str(len(genomeIds)) + '\t' +
                       str(len(markerGenes)) + '\t' + str(len(colocatedSets)))
            for cs in colocatedSets:
                fout.write('\t' + ', '.join(cs))
            fout.write('\n')

        fout.close()
    def run(self, ubiquityThreshold, singleCopyThreshold, minGenomes, minMarkers, mostSpecificRank, distThreshold, genomeThreshold):
        img = IMG()
        markerset = MarkerSet()

        lineages = img.lineagesSorted(mostSpecificRank)

        fout = open('./data/colocated.tsv', 'w', 1)
        fout.write('Lineage\t# genomes\t# markers\t# co-located sets\tCo-located markers\n')

        lineageCount = 0
        for lineage in lineages:
            lineageCount += 1

            genomeIds = img.genomeIdsByTaxonomy(lineage, 'Final')
            if len(genomeIds) < minGenomes:
                continue

            countTable = img.countTable(genomeIds)
            markerGenes = markerset.markerGenes(genomeIds, countTable, ubiquityThreshold*len(genomeIds), singleCopyThreshold*len(genomeIds))

            geneDistTable = img.geneDistTable(genomeIds, markerGenes, spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable, distThreshold, genomeThreshold)
            colocatedSets = markerset.colocatedSets(colocatedGenes, markerGenes)
            if len(colocatedSets) < minMarkers:
                continue

            print '\nLineage ' + lineage + ' contains ' + str(len(genomeIds)) + ' genomes (' + str(lineageCount) + ' of ' + str(len(lineages)) + ').'
            print '  Marker genes: ' + str(len(markerGenes))
            print '  Co-located gene sets: ' + str(len(colocatedSets))

            fout.write(lineage + '\t' + str(len(genomeIds)) + '\t' + str(len(markerGenes)) + '\t' + str(len(colocatedSets)))
            for cs in colocatedSets:
                fout.write('\t' + ', '.join(cs))
            fout.write('\n')

        fout.close()
    def run(self, ubiquityThreshold, singleCopyThreshold, minGenomes,
            mostSpecificRank, minMarkers, completenessThreshold,
            contaminationThreshold):
        print 'Ubiquity threshold: ' + str(ubiquityThreshold)
        print 'Single-copy threshold: ' + str(singleCopyThreshold)
        print 'Min. genomes: ' + str(minGenomes)
        print 'Most specific taxonomic rank: ' + str(mostSpecificRank)
        print 'Min markers: ' + str(minMarkers)
        print 'Completeness threshold: ' + str(completenessThreshold)
        print 'Contamination threshold: ' + str(contaminationThreshold)

        img = IMG()
        markerset = MarkerSet()

        lineages = img.lineagesByCriteria(minGenomes, mostSpecificRank)

        degenerateGenomes = {}
        for lineage in lineages:
            genomeIds = img.genomeIdsByTaxonomy(lineage, 'Final')

            print ''
            print 'Lineage ' + lineage + ' contains ' + str(
                len(genomeIds)) + ' genomes.'

            # get table of PFAMs and do some initial filtering to remove PFAMs that are
            # clearly not going to pass the ubiquity and single-copy thresholds
            countTable = img.countTable(genomeIds)
            countTable = img.filterTable(genomeIds, countTable,
                                         ubiquityThreshold * 0.9,
                                         singleCopyThreshold * 0.9)

            markerGenes = markerset.markerGenes(
                genomeIds, countTable, ubiquityThreshold * len(genomeIds),
                singleCopyThreshold * len(genomeIds))
            if len(markerGenes) < minMarkers:
                continue

            geneDistTable = img.geneDistTable(genomeIds,
                                              markerGenes,
                                              spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable)
            colocatedSets = markerset.colocatedSets(colocatedGenes,
                                                    markerGenes)

            for genomeId in genomeIds:
                completeness, contamination = markerset.genomeCheck(
                    colocatedSets, genomeId, countTable)

                if completeness < completenessThreshold or contamination > contaminationThreshold:
                    degenerateGenomes[genomeId] = degenerateGenomes.get(
                        genomeId, []) + [[
                            lineage.split(';')[-1].strip(),
                            len(genomeIds),
                            len(colocatedSets), completeness, contamination
                        ]]

        # write out degenerate genomes
        metadata = img.genomeMetadata('Final')

        fout = open('./data/degenerate_genomes.tsv', 'w')
        fout.write(
            'Genome Id\tTaxonomy\tGenome Size (Gbps)\tScaffolds\tBiotic Relationships\tStatus\tLineage\t# genomes\tMarker set size\tCompleteness\tContamination\n'
        )
        for genomeId, data in degenerateGenomes.iteritems():
            fout.write(genomeId + '\t' +
                       '; '.join(metadata[genomeId]['taxonomy']) + '\t%.2f' %
                       (float(metadata[genomeId]['genome size']) / 1e6) +
                       '\t' + str(metadata[genomeId]['scaffold count']))
            fout.write('\t' + metadata[genomeId]['biotic relationships'] +
                       '\t' + metadata[genomeId]['status'])

            for d in data:
                fout.write('\t' + d[0] + '\t' + str(d[1]) + '\t' + str(d[2]) +
                           '\t%.3f\t%.3f' % (d[3], d[4]))
            fout.write('\n')

        fout.close()
    def run(self, ubiquityThreshold, singleCopyThreshold, trustedCompleteness, trustedContamination, genomeCompleteness, genomeContamination):
        img = IMG()
        markerset = MarkerSet()

        metadata = img.genomeMetadata()

        trustedOut = open('./data/trusted_genomes.tsv', 'w')
        trustedOut.write('Genome Id\tLineage\tGenome size (Mbps)\tScaffold count\tBiotic Relationship\tStatus\tCompleteness\tContamination\n')

        filteredOut = open('./data/filtered_genomes.tsv', 'w')
        filteredOut.write('Genome Id\tLineage\tGenome size (Mbps)\tScaffold count\tBiotic Relationship\tStatus\tCompleteness\tContamination\n')

        allGenomeIds = set()
        allTrustedGenomeIds = set()
        for lineage in ['Archaea', 'Bacteria']:
            # get all genomes in lineage and build gene count table
            print '\nBuilding gene count table.'
            allLineageGenomeIds = img.genomeIdsByTaxonomy(lineage, metadata, 'All')
            countTable = img.countTable(allLineageGenomeIds)
            countTable = img.filterTable(allLineageGenomeIds, countTable, 0.9*ubiquityThreshold, 0.9*singleCopyThreshold)

            # get all genomes from specific lineage
            allGenomeIds = allGenomeIds.union(allLineageGenomeIds)

            print 'Lineage ' + lineage + ' contains ' + str(len(allLineageGenomeIds)) + ' genomes.'

            # tabulate genomes from each phylum
            allPhylumCounts = {}
            for genomeId in allLineageGenomeIds:
                taxon = metadata[genomeId]['taxonomy'][1]
                allPhylumCounts[taxon] = allPhylumCounts.get(taxon, 0) + 1

            # identify marker set for genomes
            markerGenes = markerset.markerGenes(allLineageGenomeIds, countTable, ubiquityThreshold*len(allLineageGenomeIds), singleCopyThreshold*len(allLineageGenomeIds))
            print '  Marker genes: ' + str(len(markerGenes))

            geneDistTable = img.geneDistTable(allLineageGenomeIds, markerGenes, spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable, metadata)
            colocatedSets = markerset.colocatedSets(colocatedGenes, markerGenes)
            print '  Marker set size: ' + str(len(colocatedSets))

            # identifying trusted genomes (highly complete, low contamination genomes)
            trustedGenomeIds = set()
            for genomeId in allLineageGenomeIds:
                completeness, contamination = markerset.genomeCheck(colocatedSets, genomeId, countTable)

                if completeness >= trustedCompleteness and contamination <= trustedContamination:
                    trustedGenomeIds.add(genomeId)
                    allTrustedGenomeIds.add(genomeId)

                    trustedOut.write(genomeId + '\t' + '; '.join(metadata[genomeId]['taxonomy']))
                    trustedOut.write('\t%.2f' % (float(metadata[genomeId]['genome size']) / 1e6))
                    trustedOut.write('\t' + str(metadata[genomeId]['scaffold count']))
                    trustedOut.write('\t' + metadata[genomeId]['biotic relationships'])
                    trustedOut.write('\t' + metadata[genomeId]['status'])
                    trustedOut.write('\t%.3f\t%.3f' % (completeness, contamination) + '\n')
                else:
                    filteredOut.write(genomeId + '\t' + '; '.join(metadata[genomeId]['taxonomy']))
                    filteredOut.write('\t%.2f' % (float(metadata[genomeId]['genome size']) / 1e6))
                    filteredOut.write('\t' + str(metadata[genomeId]['scaffold count']))
                    filteredOut.write('\t' + metadata[genomeId]['biotic relationships'])
                    filteredOut.write('\t' + metadata[genomeId]['status'])
                    filteredOut.write('\t%.3f\t%.3f' % (completeness, contamination) + '\n')

            print '  Trusted genomes: ' + str(len(trustedGenomeIds))

            # determine status of trusted genomes
            statusBreakdown = {}
            for genomeId in trustedGenomeIds:
                statusBreakdown[metadata[genomeId]['status']] = statusBreakdown.get(metadata[genomeId]['status'], 0) + 1

            print '  Trusted genome status breakdown: '
            for status, count in statusBreakdown.iteritems():
                print '    ' + status + ': ' + str(count)

            # determine status of retained genomes
            proposalNameBreakdown = {}
            for genomeId in trustedGenomeIds:
                proposalNameBreakdown[metadata[genomeId]['proposal name']] = proposalNameBreakdown.get(metadata[genomeId]['proposal name'], 0) + 1

            print '  Retained genome proposal name breakdown: '
            for pn, count in proposalNameBreakdown.iteritems():
                if 'KMG' in pn or 'GEBA' in pn or 'HMP' in pn:
                    print '    ' + pn + ': ' + str(count)

            print '  Filtered genomes by phylum:'
            trustedPhylumCounts = {}
            for genomeId in trustedGenomeIds:
                taxon = metadata[genomeId]['taxonomy'][1]
                trustedPhylumCounts[taxon] = trustedPhylumCounts.get(taxon, 0) + 1

            for phylum, count in allPhylumCounts.iteritems():
                print phylum + ': %d of %d' % (trustedPhylumCounts.get(phylum, 0), count)

        trustedOut.close()
        filteredOut.close()

        # write out lineage statistics for genome distribution
        allStats = {}
        trustedStats = {}

        for r in xrange(0, 6): # Domain to Genus
            for genomeId, data in metadata.iteritems():
                taxaStr = '; '.join(data['taxonomy'][0:r+1])
                allStats[taxaStr] = allStats.get(taxaStr, 0) + 1
                if genomeId in allTrustedGenomeIds:
                    trustedStats[taxaStr] = trustedStats.get(taxaStr, 0) + 1

        sortedLineages = img.lineagesSorted()

        fout = open('./data/lineage_stats.tsv', 'w')
        fout.write('Lineage\tGenomes with metadata\tTrusted genomes\n')
        for lineage in sortedLineages:
            fout.write(lineage + '\t' + str(allStats.get(lineage, 0))+ '\t' + str(trustedStats.get(lineage, 0))+ '\n')
        fout.close()
Example #7
0
    def run(self, ubiquityThreshold, singleCopyThreshold, trustedCompleteness,
            trustedContamination, genomeCompleteness, genomeContamination):
        img = IMG()
        markerset = MarkerSet()

        metadata = img.genomeMetadata()

        trustedOut = open('./data/trusted_genomes.tsv', 'w')
        trustedOut.write(
            'Genome Id\tLineage\tGenome size (Mbps)\tScaffold count\tBiotic Relationship\tStatus\tCompleteness\tContamination\n'
        )

        filteredOut = open('./data/filtered_genomes.tsv', 'w')
        filteredOut.write(
            'Genome Id\tLineage\tGenome size (Mbps)\tScaffold count\tBiotic Relationship\tStatus\tCompleteness\tContamination\n'
        )

        allGenomeIds = set()
        allTrustedGenomeIds = set()
        for lineage in ['Archaea', 'Bacteria']:
            # get all genomes in lineage and build gene count table
            print '\nBuilding gene count table.'
            allLineageGenomeIds = img.genomeIdsByTaxonomy(
                lineage, metadata, 'All')
            countTable = img.countTable(allLineageGenomeIds)
            countTable = img.filterTable(allLineageGenomeIds, countTable,
                                         0.9 * ubiquityThreshold,
                                         0.9 * singleCopyThreshold)

            # get all genomes from specific lineage
            allGenomeIds = allGenomeIds.union(allLineageGenomeIds)

            print 'Lineage ' + lineage + ' contains ' + str(
                len(allLineageGenomeIds)) + ' genomes.'

            # tabulate genomes from each phylum
            allPhylumCounts = {}
            for genomeId in allLineageGenomeIds:
                taxon = metadata[genomeId]['taxonomy'][1]
                allPhylumCounts[taxon] = allPhylumCounts.get(taxon, 0) + 1

            # identify marker set for genomes
            markerGenes = markerset.markerGenes(
                allLineageGenomeIds, countTable,
                ubiquityThreshold * len(allLineageGenomeIds),
                singleCopyThreshold * len(allLineageGenomeIds))
            print '  Marker genes: ' + str(len(markerGenes))

            geneDistTable = img.geneDistTable(allLineageGenomeIds,
                                              markerGenes,
                                              spacingBetweenContigs=1e6)
            colocatedGenes = markerset.colocatedGenes(geneDistTable, metadata)
            colocatedSets = markerset.colocatedSets(colocatedGenes,
                                                    markerGenes)
            print '  Marker set size: ' + str(len(colocatedSets))

            # identifying trusted genomes (highly complete, low contamination genomes)
            trustedGenomeIds = set()
            for genomeId in allLineageGenomeIds:
                completeness, contamination = markerset.genomeCheck(
                    colocatedSets, genomeId, countTable)

                if completeness >= trustedCompleteness and contamination <= trustedContamination:
                    trustedGenomeIds.add(genomeId)
                    allTrustedGenomeIds.add(genomeId)

                    trustedOut.write(genomeId + '\t' +
                                     '; '.join(metadata[genomeId]['taxonomy']))
                    trustedOut.write(
                        '\t%.2f' %
                        (float(metadata[genomeId]['genome size']) / 1e6))
                    trustedOut.write('\t' +
                                     str(metadata[genomeId]['scaffold count']))
                    trustedOut.write(
                        '\t' + metadata[genomeId]['biotic relationships'])
                    trustedOut.write('\t' + metadata[genomeId]['status'])
                    trustedOut.write('\t%.3f\t%.3f' %
                                     (completeness, contamination) + '\n')
                else:
                    filteredOut.write(
                        genomeId + '\t' +
                        '; '.join(metadata[genomeId]['taxonomy']))
                    filteredOut.write(
                        '\t%.2f' %
                        (float(metadata[genomeId]['genome size']) / 1e6))
                    filteredOut.write(
                        '\t' + str(metadata[genomeId]['scaffold count']))
                    filteredOut.write(
                        '\t' + metadata[genomeId]['biotic relationships'])
                    filteredOut.write('\t' + metadata[genomeId]['status'])
                    filteredOut.write('\t%.3f\t%.3f' %
                                      (completeness, contamination) + '\n')

            print '  Trusted genomes: ' + str(len(trustedGenomeIds))

            # determine status of trusted genomes
            statusBreakdown = {}
            for genomeId in trustedGenomeIds:
                statusBreakdown[metadata[genomeId]
                                ['status']] = statusBreakdown.get(
                                    metadata[genomeId]['status'], 0) + 1

            print '  Trusted genome status breakdown: '
            for status, count in statusBreakdown.iteritems():
                print '    ' + status + ': ' + str(count)

            # determine status of retained genomes
            proposalNameBreakdown = {}
            for genomeId in trustedGenomeIds:
                proposalNameBreakdown[metadata[genomeId][
                    'proposal name']] = proposalNameBreakdown.get(
                        metadata[genomeId]['proposal name'], 0) + 1

            print '  Retained genome proposal name breakdown: '
            for pn, count in proposalNameBreakdown.iteritems():
                if 'KMG' in pn or 'GEBA' in pn or 'HMP' in pn:
                    print '    ' + pn + ': ' + str(count)

            print '  Filtered genomes by phylum:'
            trustedPhylumCounts = {}
            for genomeId in trustedGenomeIds:
                taxon = metadata[genomeId]['taxonomy'][1]
                trustedPhylumCounts[taxon] = trustedPhylumCounts.get(taxon,
                                                                     0) + 1

            for phylum, count in allPhylumCounts.iteritems():
                print phylum + ': %d of %d' % (trustedPhylumCounts.get(
                    phylum, 0), count)

        trustedOut.close()
        filteredOut.close()

        # write out lineage statistics for genome distribution
        allStats = {}
        trustedStats = {}

        for r in xrange(0, 6):  # Domain to Genus
            for genomeId, data in metadata.iteritems():
                taxaStr = '; '.join(data['taxonomy'][0:r + 1])
                allStats[taxaStr] = allStats.get(taxaStr, 0) + 1
                if genomeId in allTrustedGenomeIds:
                    trustedStats[taxaStr] = trustedStats.get(taxaStr, 0) + 1

        sortedLineages = img.lineagesSorted()

        fout = open('./data/lineage_stats.tsv', 'w')
        fout.write('Lineage\tGenomes with metadata\tTrusted genomes\n')
        for lineage in sortedLineages:
            fout.write(lineage + '\t' + str(allStats.get(lineage, 0)) + '\t' +
                       str(trustedStats.get(lineage, 0)) + '\n')
        fout.close()