Example #1
0
 def _sort_cogs(cogs1, cogs2):
     cogs1 = cogs1[1] # discard seed info
     cogs2 = cogs2[1] # discard seed info        
     cog_sizes1 = [len(cog) for cog in cogs1]
     cog_sizes2 = [len(cog) for cog in cogs2]
     mx1, mn1, avg1 = _max(cog_sizes1), _min(cog_sizes1), round(_mean(cog_sizes1))
     mx2, mn2, avg2 = _max(cog_sizes2), _min(cog_sizes2), round(_mean(cog_sizes2))
     
     # we want to maximize all these values in the following order:
     for i, j in ((mx1, mx2), (avg1, avg2), (len(cogs1), len(cogs2))):
         v = -1 * cmp(i, j)
         if v != 0:
             break
     return v
Example #2
0
    def _sort_cogs(cogs1, cogs2):
        cogs1 = cogs1[1]  # discard seed info
        cogs2 = cogs2[1]  # discard seed info
        cog_sizes1 = [len(cog) for cog in cogs1]
        cog_sizes2 = [len(cog) for cog in cogs2]
        mx1, mn1, avg1 = _max(cog_sizes1), _min(cog_sizes1), round(
            _mean(cog_sizes1))
        mx2, mn2, avg2 = _max(cog_sizes2), _min(cog_sizes2), round(
            _mean(cog_sizes2))

        # we want to maximize all these values in the following order:
        for i, j in ((mx1, mx2), (avg1, avg2), (len(cogs1), len(cogs2))):
            v = -1 * cmp(i, j)
            if v != 0:
                break
        return v
Example #3
0
def get_identity(fname):
    s = SeqGroup(fname)
    seqlen = len(s.id2seq.itervalues().next())
    ident = list()
    for i in xrange(seqlen):
        states = defaultdict(int)
        for seq in s.id2seq.itervalues():
            if seq[i] != "-":
                states[seq[i]] += 1
        values = states.values()
        if values:
            ident.append(float(max(values)) / sum(values))
    return (_max(ident), _min(ident), _mean(ident), _std(ident))
Example #4
0
def get_seqs_identity(alg, seqs):
    """ Returns alg statistics regarding a set of sequences"""
    seqlen = len(alg.get_seq(seqs[0]))
    ident = list()
    for i in xrange(seqlen):
        states = defaultdict(int)
        for seq_id in seqs:
            seq = alg.get_seq(seq_id)
            if seq[i] != "-":
                states[seq[i]] += 1
        values = states.values()
        if values:
            ident.append(float(max(values)) / sum(values))
    return (_max(ident), _min(ident), _mean(ident), _std(ident))
Example #5
0
def get_best_selection(cogs_selections, species):
    ALL_SPECIES = set(species)

    def _compare_cog_selection(cs1, cs2):
        seed_1, missing_sp_allowed_1, candidates_1, sp2hits_1 = cs1
        seed_2, missing_sp_allowed_2, candidates_2, sp2hits_2 = cs2

        score_1, min_cov_1, max_cov_1, median_cov_1, cov_std_1, cog_cov_1 = get_cog_score(
            candidates_1, sp2hits_1, median_cogs, ALL_SPECIES - set([seed_1]))
        score_2, min_cov_2, max_cov_2, median_cov_2, cov_std_2, cog_cov_2 = get_cog_score(
            candidates_2, sp2hits_2, median_cogs, ALL_SPECIES - set([seed_2]))

        sp_represented_1 = len(sp2hits_1)
        sp_represented_2 = len(sp2hits_1)
        cmp_rpr = cmp(sp_represented_1, sp_represented_2)
        if cmp_rpr == 1:
            return 1
        elif cmp_rpr == -1:
            return -1
        else:
            cmp_score = cmp(score_1, score_2)
            if cmp_score == 1:
                return 1
            elif cmp_score == -1:
                return -1
            else:
                cmp_mincov = cmp(min_cov_1, min_cov_2)
                if cmp_mincov == 1:
                    return 1
                elif cmp_mincov == -1:
                    return -1
                else:
                    cmp_maxcov = cmp(max_cov_1, max_cov_2)
                    if cmp_maxcov == 1:
                        return 1
                    elif cmp_maxcov == -1:
                        return -1
                    else:
                        cmp_cand = cmp(len(candidates_1), len(candidates_2))
                        if cmp_cand == 1:
                            return 1
                        elif cmp_cand == -1:
                            return -1
                        else:
                            return 0

    min_score = 0.5
    max_cogs = _max([len(data[2]) for data in cogs_selections])
    median_cogs = _median([len(data[2]) for data in cogs_selections])

    cogs_selections.sort(_compare_cog_selection)
    cogs_selections.reverse()

    header = [
        'seed', 'missing sp allowed', 'spcs covered', '#COGs',
        'mean sp coverage)', '#COGs for worst sp.', '#COGs for best sp.',
        'sp. in COGS(avg)', 'SCORE'
    ]
    print_header = True
    best_cog_selection = None
    cog_analysis = StringIO()
    for i, cogs in enumerate(cogs_selections):
        seed, missing_sp_allowed, candidates, sp2hits = cogs
        sp_percent_coverages = [
            (100 * sp2hits.get(sp, 0)) / float(len(candidates))
            for sp in species
        ]
        sp_coverages = [sp2hits.get(sp, 0) for sp in species]
        score, min_cov, max_cov, median_cov, cov_std, cog_cov = get_cog_score(
            candidates, sp2hits, median_cogs, ALL_SPECIES - set([seed]))

        if best_cog_selection is None:
            best_cog_selection = i
            flag = "*"
        else:
            flag = " "
        data = (candidates,
                flag+"%10s" %seed, \
                    missing_sp_allowed, \
                    "%d (%0.1f%%)" %(len(set(sp2hits.keys()))+1, 100*float(len(ALL_SPECIES))/(len(sp2hits)+1)) , \
                    len(candidates), \
                    "%0.1f%% +- %0.1f" %(_mean(sp_percent_coverages), _std(sp_percent_coverages)), \
                    "% 3d (%0.1f%%)" %(min(sp_coverages),100*min(sp_coverages)/float(len(candidates))), \
                    "% 3d (%0.1f%%)" %(max(sp_coverages),100*max(sp_coverages)/float(len(candidates))), \
                    cog_cov,
                    score
                )
        if print_header:
            print_as_table([data[1:]],
                           header=header,
                           print_header=True,
                           stdout=cog_analysis)
            print_header = False
        else:
            print_as_table([data[1:]],
                           header=header,
                           print_header=False,
                           stdout=cog_analysis)

    #raw_input("Press")
    print cog_analysis.getvalue()
    #best_cog_selection = int(raw_input("choose:"))
    return cogs_selections[best_cog_selection], cog_analysis
Example #6
0
def brh_cogs2(DB, species, missing_factor=0.0, seed_sp=None, min_score=0):
    """It scans all precalculate BRH relationships among the species
       passed as an argument, and detects Clusters of Orthologs
       according to several criteria:

       min_score: the min coverage/overalp value required for a
       blast to be a reliable hit.

       missing_factor: the min percentage of species in which a
       given seq must have  orthologs.

    """
    def _sort_cogs(cogs1, cogs2):
        seed1, mx1, avg1, ncogs1 = cogs1
        seed2, mx2, avg2, ncogs2 = cogs2
        for i, j in ((mx1, mx2), (avg1, avg2), (ncogs1, ncogs2)):
            v = -1 * cmp(i, j)
            if v != 0:
                break
        return v

    log.log(26, "Searching BRH orthologs")
    species = set(map(str, species))

    min_species = len(species) - round(missing_factor * len(species))

    if seed_sp == "auto":
        sp_to_test = list(species)
    elif seed_sp == "largest":
        cmd = """SELECT taxid, size FROM species"""
        db.seqcursor.execute(cmd)
        sp2size = {}
        for tax, counter in db.seqcursor.fetchall():
            if tax in species:
                sp2size[tax] = counter

        sorted_sp = sorted(sp2size.items(), lambda x, y: cmp(x[1], y[1]))
        log.log(24, sorted_sp[:6])
        largest_sp = sorted_sp[-1][0]
        sp_to_test = [largest_sp]
        log.log(28, "Using %s as search seed. Proteome size=%s genes" %\
            (largest_sp, sp2size[largest_sp]))
    else:
        sp_to_test = [str(seed_sp)]

    analysis_txt = StringIO()
    if sp_to_test:
        log.log(26, "Finding best COG selection...")
        seed2size = get_sorted_seeds(seed_sp, species, sp_to_test, min_species,
                                     DB)
        size_analysis = []
        for seedname, content in seed2size.iteritems():
            cog_sizes = [size for seq, size in content]
            mx, avg = _max(cog_sizes), round(_mean(cog_sizes))
            size_analysis.append([seedname, mx, avg, len(content)])
        size_analysis.sort(_sort_cogs)
        #print '\n'.join(map(str, size_analysis))
        seed = size_analysis[0][0]
        print_as_table(
            size_analysis[:25],
            stdout=analysis_txt,
            header=["Seed", "largest COG", "avg COG size", "total COGs"])
        if size_analysis[0][1] < len(species) - 1:
            print size_analysis[0][1]
            raise ValueError(
                "Current COG selection parameters do not permit to cover all species"
            )

    log.log(28, analysis_txt.getvalue())
    # The following loop tests each possible seed if none is
    # specified.
    log.log(28, "Computing Clusters of Orthologs groups (COGs)")
    log.log(28, "Min number of species per COG: %d" % min_species)
    cogs_selection = []
    log.log(26, "Using seed species:%s", seed)
    species_side1 = ','.join(
        map(quote, [s for s in species if str(s) > str(seed)]))
    species_side2 = ','.join(
        map(quote, [s for s in species if str(s) < str(seed)]))
    pairs1 = []
    pairs2 = []
    # Select all ids with matches in the target species, and
    # return the total number of species covered by each of
    # such ids.
    if species_side1 != "":
        cmd = """SELECT seqid1, taxid1, seqid2, taxid2 from ortho_pair WHERE
            taxid1="%s" AND taxid2 IN (%s) """ % (seed, species_side1)
        DB.orthocursor.execute(cmd)
        pairs1 = DB.orthocursor.fetchall()

    if species_side2 != "":
        cmd = """SELECT seqid2, taxid2, seqid1, taxid1 from ortho_pair WHERE
            taxid1 IN (%s) AND taxid2 = "%s" """ % (species_side2, seed)
        DB.orthocursor.execute(cmd)
        pairs2 = DB.orthocursor.fetchall()

    cog_candidates = defaultdict(set)
    for seq1, sp1, seq2, sp2 in pairs1 + pairs2:
        s1 = (sp1, seq1)
        s2 = (sp2, seq2)
        cog_candidates[(sp1, seq1)].update([s1, s2])

    all_cogs = [
        cand for cand in cog_candidates.values() if len(cand) >= min_species
    ]

    # CHECK CONSISTENCY
    seqs = set()
    for cand in all_cogs:
        seqs.update([b for a, b in cand if a == seed])
    pre_selected_seqs = set([v[0] for v in seed2size[seed]])
    if len(seqs & pre_selected_seqs) != len(set(seed2size[seed])) or\
            len(seqs & pre_selected_seqs) != len(seqs):
        print "old method seqs", len(seqs), "new seqs", len(
            set(seed2size[seed])), "Common", len(seqs & pre_selected_seqs)
        raise ValueError("ooops")

    cog_sizes = [len(cog) for cog in all_cogs]
    cog_spsizes = [len(set([e[0] for e in cog])) for cog in all_cogs]

    if [1 for i in xrange(len(cog_sizes)) if cog_sizes[i] != cog_spsizes[i]]:
        raise ValueError("Inconsistent COG found")

    if cog_sizes:
        cogs_selection.append([seed, all_cogs])
    log.log(26, "Found %d COGs" % len(all_cogs))

    recoded_cogs = []
    for cog in all_cogs:
        named_cog = map(
            lambda x: "%s%s%s" % (x[0], GLOBALS["spname_delimiter"], x[1]),
            cog)
        recoded_cogs.append(named_cog)

    return recoded_cogs, analysis_txt.getvalue()
Example #7
0
def get_best_selection(cogs_selections, species):
    ALL_SPECIES = set(species)
    
    def _compare_cog_selection(cs1, cs2):
        seed_1, missing_sp_allowed_1, candidates_1, sp2hits_1 = cs1
        seed_2, missing_sp_allowed_2, candidates_2, sp2hits_2 = cs2

        score_1, min_cov_1, max_cov_1, median_cov_1, cov_std_1, cog_cov_1 = get_cog_score(candidates_1, sp2hits_1, median_cogs, ALL_SPECIES-set([seed_1]))
        score_2, min_cov_2, max_cov_2, median_cov_2, cov_std_2, cog_cov_2 = get_cog_score(candidates_2, sp2hits_2, median_cogs, ALL_SPECIES-set([seed_2]))

        sp_represented_1 = len(sp2hits_1)
        sp_represented_2 = len(sp2hits_1)
        cmp_rpr = cmp(sp_represented_1, sp_represented_2)
        if cmp_rpr == 1:
            return 1
        elif cmp_rpr == -1:
            return -1
        else:
            cmp_score = cmp(score_1, score_2)
            if cmp_score == 1:
                return 1
            elif cmp_score == -1:
                return -1
            else:
                cmp_mincov = cmp(min_cov_1, min_cov_2)
                if cmp_mincov == 1: 
                    return 1
                elif cmp_mincov == -1: 
                    return -1 
                else:
                    cmp_maxcov = cmp(max_cov_1, max_cov_2)
                    if cmp_maxcov == 1: 
                        return 1
                    elif cmp_maxcov == -1: 
                        return -1 
                    else:
                        cmp_cand = cmp(len(candidates_1), len(candidates_2))
                        if cmp_cand == 1:
                            return 1
                        elif cmp_cand == -1:
                            return -1
                        else:
                            return 0 
    
    min_score = 0.5
    max_cogs = _max([len(data[2]) for data in cogs_selections])
    median_cogs = _median([len(data[2]) for data in cogs_selections])

    cogs_selections.sort(_compare_cog_selection)            
    cogs_selections.reverse()

    header = ['seed',
              'missing sp allowed',
              'spcs covered',
              '#COGs',
              'mean sp coverage)',
              '#COGs for worst sp.',
              '#COGs for best sp.',
              'sp. in COGS(avg)',
              'SCORE' ]
    print_header = True
    best_cog_selection = None
    cog_analysis = StringIO()
    for i, cogs in enumerate(cogs_selections):
        seed, missing_sp_allowed, candidates, sp2hits = cogs
        sp_percent_coverages = [(100*sp2hits.get(sp,0))/float(len(candidates)) for sp in species]
        sp_coverages = [sp2hits.get(sp, 0) for sp in species]
        score, min_cov, max_cov, median_cov, cov_std, cog_cov = get_cog_score(candidates, sp2hits, median_cogs, ALL_SPECIES-set([seed]))

        if best_cog_selection is None:
            best_cog_selection = i
            flag = "*"
        else:
            flag = " "
        data = (candidates, 
                flag+"%10s" %seed, \
                    missing_sp_allowed, \
                    "%d (%0.1f%%)" %(len(set(sp2hits.keys()))+1, 100*float(len(ALL_SPECIES))/(len(sp2hits)+1)) , \
                    len(candidates), \
                    "%0.1f%% +- %0.1f" %(_mean(sp_percent_coverages), _std(sp_percent_coverages)), \
                    "% 3d (%0.1f%%)" %(min(sp_coverages),100*min(sp_coverages)/float(len(candidates))), \
                    "% 3d (%0.1f%%)" %(max(sp_coverages),100*max(sp_coverages)/float(len(candidates))), \
                    cog_cov,
                    score         
                )
        if print_header:
            print_as_table([data[1:]], header=header, print_header=True, stdout=cog_analysis)
            print_header = False
        else:
            print_as_table([data[1:]], header=header, print_header=False, stdout=cog_analysis)

    #raw_input("Press")
    print cog_analysis.getvalue()
    #best_cog_selection = int(raw_input("choose:"))
    return cogs_selections[best_cog_selection], cog_analysis
Example #8
0
def brh_cogs2(DB, species, missing_factor=0.0, seed_sp=None, min_score=0):
    """It scans all precalculate BRH relationships among the species
       passed as an argument, and detects Clusters of Orthologs
       according to several criteria:

       min_score: the min coverage/overalp value required for a
       blast to be a reliable hit.

       missing_factor: the min percentage of species in which a
       given seq must have  orthologs.

    """
    def _sort_cogs(cogs1, cogs2):
        seed1, mx1, avg1, ncogs1 = cogs1
        seed2, mx2, avg2, ncogs2 = cogs2
        for i, j in ((mx1, mx2), (avg1, avg2), (ncogs1, ncogs2)):
            v = -1 * cmp(i, j)
            if v != 0:
                break
        return v
    
    log.log(26, "Searching BRH orthologs")
    species = set(map(str, species))
    
    min_species = len(species) - round(missing_factor * len(species))
    
    if seed_sp == "auto":
        sp_to_test = list(species)
    elif seed_sp == "largest":
        cmd = """SELECT taxid, size FROM species"""
        db.seqcursor.execute(cmd)
        sp2size = {}
        for tax, counter in db.seqcursor.fetchall():
            if tax in species: 
                sp2size[tax] = counter
            
        sorted_sp = sorted(sp2size.items(), lambda x,y: cmp(x[1],y[1]))
        log.log(24, sorted_sp[:6])
        largest_sp = sorted_sp[-1][0]
        sp_to_test = [largest_sp]
        log.log(28, "Using %s as search seed. Proteome size=%s genes" %\
            (largest_sp, sp2size[largest_sp]))
    else:
        sp_to_test = [str(seed_sp)]

    analysis_txt = StringIO()
    if sp_to_test:
        log.log(26, "Finding best COG selection...")
        seed2size = get_sorted_seeds(seed_sp, species, sp_to_test, min_species, DB)
        size_analysis = []
        for seedname, content in seed2size.iteritems():
            cog_sizes = [size for seq, size in content]
            mx, avg = _max(cog_sizes), round(_mean(cog_sizes))
            size_analysis.append([seedname, mx, avg, len(content)])
        size_analysis.sort(_sort_cogs)                
        #print '\n'.join(map(str, size_analysis))
        seed = size_analysis[0][0]
        print_as_table(size_analysis[:25], stdout=analysis_txt,
                   header=["Seed","largest COG", "avg COG size", "total COGs"])
        if size_analysis[0][1] < len(species)-1:
            print size_analysis[0][1]
            raise ValueError("Current COG selection parameters do not permit to cover all species")
       
    log.log(28, analysis_txt.getvalue())
    # The following loop tests each possible seed if none is
    # specified.
    log.log(28, "Computing Clusters of Orthologs groups (COGs)")
    log.log(28, "Min number of species per COG: %d" %min_species)
    cogs_selection = []
    log.log(26,"Using seed species:%s", seed)
    species_side1 = ','.join(map(quote, [s for s in species if str(s)>str(seed)]))
    species_side2 = ','.join(map(quote, [s for s in species if str(s)<str(seed)]))
    pairs1 = []
    pairs2 = []
    # Select all ids with matches in the target species, and
    # return the total number of species covered by each of
    # such ids.
    if species_side1 != "":
        cmd = """SELECT seqid1, taxid1, seqid2, taxid2 from ortho_pair WHERE
            taxid1="%s" AND taxid2 IN (%s) """ % (seed, species_side1)
        DB.orthocursor.execute(cmd)
        pairs1 = DB.orthocursor.fetchall()

    if species_side2 != "":
        cmd = """SELECT seqid2, taxid2, seqid1, taxid1 from ortho_pair WHERE
            taxid1 IN (%s) AND taxid2 = "%s" """ % (species_side2, seed)
        DB.orthocursor.execute(cmd)
        pairs2 = DB.orthocursor.fetchall()
        
    cog_candidates = defaultdict(set)
    for seq1, sp1, seq2, sp2 in pairs1 + pairs2:
        s1 = (sp1, seq1)
        s2 = (sp2, seq2)
        cog_candidates[(sp1, seq1)].update([s1, s2])

    all_cogs = [cand for cand in cog_candidates.values() if
                len(cand) >= min_species]

    # CHECK CONSISTENCY
    seqs = set()
    for cand in all_cogs:
        seqs.update([b for a,b  in cand if a == seed])
    pre_selected_seqs = set([v[0] for v in seed2size[seed]])
    if len(seqs & pre_selected_seqs) != len(set(seed2size[seed])) or\
            len(seqs & pre_selected_seqs) != len(seqs): 
        print "old method seqs", len(seqs), "new seqs", len(set(seed2size[seed])), "Common", len(seqs & pre_selected_seqs)
        raise ValueError("ooops")
        
    cog_sizes = [len(cog) for cog in all_cogs]
    cog_spsizes = [len(set([e[0] for e in cog])) for cog in all_cogs]

    if [1 for i in xrange(len(cog_sizes)) if cog_sizes[i] != cog_spsizes[i]]:
        raise ValueError("Inconsistent COG found")
            
    if cog_sizes: 
        cogs_selection.append([seed, all_cogs])
    log.log(26, "Found %d COGs" % len(all_cogs))
    
    recoded_cogs = []
    for cog in all_cogs:
        named_cog = map(lambda x: "%s%s%s" %(x[0], GLOBALS["spname_delimiter"],x[1]), cog)
        recoded_cogs.append(named_cog)

    return recoded_cogs, analysis_txt.getvalue()