def infer_derivative(sense, main_sense_of_entry):
    equiv = None
    if (main_sense_of_entry is not None and
        not main_sense_of_entry.is_affix() and
        main_sense_of_entry.thesclass is not None):
        # Take precedent from the main sense of the parent entry
        equiv = tdb.equivalent_class(main_sense_of_entry.thesclass, sense.wordclass)
        if equiv is not None:
            equiv.reason_text = 'Parallel to "%s"' % main_sense_of_entry.lemma
            equiv.reason_code = 'driv'
    if equiv is None:
        # Take precedent from other sibling derivatives
        candidates = tdb.ranked_search(refentry=sense.entry_id,
            thes_linked=True, currentOnly=True)
        candidates = [c for c in candidates if c.is_derivative() and
                      not ' ' in c.lemma and
                      c.superclass() is not None]
        if candidates:
            j = defaultdict(int)
            for c in candidates:
                j[c.superclass()] += 1
            parents = [p for p, num in j.items() if num == max(j.values())]
            # If there's more than one possible parent, pick the one with
            #  the largest branch
            parents.sort(key=lambda p: p.branch_size, reverse=True)
            equiv = tdb.child_wordclass_branch(parents[0], sense.wordclass)
            if equiv is None:
                equiv = parents[0]
            if equiv is not None:
                equiv.reason_text = 'Parallel to %s' % ', '.join(
                    ['"%s"' % (c.lemma,) for c in candidates if c.lemma])
                equiv.reason_code = 'driv'
    return equiv
 def branches_from_xrefs(self, sense):
     branches = set()
     for xr in sense.cross_references:
         instances = tdb.ranked_search(refentry=xr.refentry, refid=xr.refid)
         local_branches = _parse_instances(instances)
         branches = branches | local_branches
     return [b.id for b in branches]
def finalize():
    """
    Use the 'manual' file to override where necessary
    """
    for wordclass in WORDCLASSES:
        lemmas = {}
        infile1 = os.path.join(DIRECTORY, '%s_compounds.csv' % wordclass)
        infile2 = os.path.join(DIRECTORY, '%s_manual.csv' % wordclass)
        with open(infile1, 'r') as filehandle:
            csvreader = csv.reader(filehandle)
            for row in csvreader:
                lemmas[row[0]] = int(row[1])
        # Do the manual file second, so that it overrides the
        #  automatically-generated file
        with open(infile2, 'r') as filehandle:
            csvreader = csv.reader(filehandle)
            for row in csvreader:
                lemmas[row[0]] = int(row[1])

        output = []
        for lemma, class_id in lemmas.items():
            # Retrieve the branch that the majority of compounds are on
            compound_branch = tdb.get_thesclass(class_id)

            # Get the highest-rated senses for the lemma
            ranked_senses = tdb.ranked_search(lemma=lemma, wordclass=wordclass)
            if ranked_senses:
                max_rating = ranked_senses[0].rating()
                ranked_senses = [s for s in ranked_senses if
                                 max_rating > 0 and
                                 s.rating() > max_rating * 0.3]

            # Try filtering to just those senses that are on
            #   the same branch as the compounds
            ranked_filtered = [s for i, s in enumerate(ranked_senses) if
                               (i == 0 and s.thesclass is None) or
                               s.is_descendant_of(compound_branch)]
            # ... or else stick with original ranking
            if not ranked_filtered:
                ranked_filtered = ranked_senses

            if ranked_filtered:
                output.append(ranked_filtered[0])

        outfile = os.path.join(DIRECTORY, '%s.csv' % wordclass)
        output.sort(key=lambda s: s.lemma)
        with open(outfile, 'w') as filehandle:
            csvwriter = csv.writer(filehandle)
            for s in output:
                row = (s.lemma, s.refentry, s.refid,
                       s.entry_size, s.breadcrumb())
                csvwriter.writerow(row)
def compound_derivative(sense):
    if sense.last_element() is None:
        return None, None

    thesclass = None
    for ending, wordclasses, replacement in derivation_forms:
        if (sense.wordclass in wordclasses and
            sense.last_element().endswith(ending) and
            len(sense.last_element()) > len(ending) + 2):

            # Figure out what the base form would look like,
            #  if the lemma *is* a derivative
            # - Strip off the ending, then add the replacement ending
            #  (which is usually a null string).
            hypothetical_base =\
                sense.lemma[0:len(sense.lemma)-len(ending)] + replacement

            # Test if the hypothetical base form exists, and if so
            #  find out how it is classified
            base_classifications = tdb.ranked_search(lemma=hypothetical_base,
                                                     current_only=True)

            if (tdb.distinct_senses(base_classifications) == 1 and
                    base_classifications[0].thesclass is not None):
                thesclass = base_classifications[0].thesclass
                break

    if thesclass is not None:
        # Don't risk things like 'yellow-bellied' - these are
        #  likely to be transparent, not a derivative of e.g.
        #  'yellow-belly', so should go after existing guesses
        if ending in ('ed', 'ied') and sense.first_element().lower() in colours:
            position = 'last'
        else:
            position = 'first'
        return base_classifications[0].thesclass, position
    else:
        return None, None
def ranked_sense_summary(**kwargs):
    """
    Return a summary of the ranked_search() results, reduced to a
    given thesaurus level (defaults to level=3)

    Returns a ranked list of result objects, each with the following
    attributes:
     * parent: level-3 parent class;
     * classes: ranked list of thesaurus classes which map to the parent;
     * summed_rating: sum of c.rating() values for each c in classes;
     * probability: probability of this row (based on summed rating).

    Keyword arguments:
     * level (int): the parent thesaurus level which will be returned
     * omit_null (True/False): if True, any senses which don't have
            a classification will be ignored
     ... plus all the usual optional keyword arguments passed on to
     tdb.search() (lemma, wordclass, refentry, refid, etc.).
    """
    level = kwargs.get('level', 3)
    lemma = kwargs.get('lemma')
    wordclass = kwargs.get('wordclass')
    omit_null = kwargs.get('omit_null', False)

    if lemma is not None:
        main_sense = main_sense_finder.main_sense(lemma=lemma,
                                                  wordclass=wordclass,
                                                  listed_only=True)
        if main_sense is not None:
            kwargs['promote'] = main_sense.refid

    candidates = tdb.ranked_search(**kwargs)
    if omit_null:
        candidates = [c for c in candidates if c.thesclass is not None]

    # Give each thesclass a probability value (as a ratio of the sum
    #  of all senses' ratings)
    total = sum([c.rating() for c in candidates])
    if total <= 0:
        total = 1

    summary = {}
    for c in candidates:
        if c.thesclass is None or c.thesclass.ancestor(level=level) is None:
            ancestor = None
            identifier = 0
        else:
            ancestor = c.thesclass.ancestor(level=level)
            identifier = ancestor.id
        if not identifier in summary:
            summary[identifier] = ResultRow(ancestor)
        summary[identifier].append(c)

    # Convert to a list
    summary = list(summary.values())
    # Add a probability score (0 < p < 1) to each row
    [row.set_probability(total) for row in summary]
    # Sort by probability
    summary.sort(key=lambda r: r.probability, reverse=True)

    return summary
def match_single_synonym(sense):
    # Drop out any highly polysemous synonyms
    synonyms = []
    for syn in sense.synonyms:
        instances = tdb.search(lemma=syn,
                               wordclass=sense.wordclass,
                               current_only=True)
        if tdb.distinct_senses(instances) < 20:
            synonyms.append(syn)

    if not synonyms:
        return None, None

    match = None
    matching_synonym = None

    # If the sense can be restricted by subject area, try to find a match for
    #  *any* synonym (even if there's only one)
    if not match and synonyms and sense.subjects:
        candidates = []
        for syn in synonyms:
            candidates.extend(tdb.ranked_search(lemma=syn,
                                                wordclass=sense.wordclass,
                                                subjects=sense.subjects,
                                                current_only=True))
        if candidates and candidates[0].thesclass is not None:
            match = candidates[0].thesclass
            matching_synonym = candidates[0].lemma

    # If the sense is an interjection, try to find a match for
    #  *any* synonym (even if there's only one) - since interjection
    #  synonyms are more reliable and less ambiguous
    if not match and synonyms and sense.wordclass == 'UH':
        candidates = []
        for syn in synonyms:
            candidates.extend(tdb.ranked_search(lemma=syn,
                                                wordclass='UH',
                                                current_only=True))
        if candidates and candidates[0].thesclass is not None:
            match = candidates[0].thesclass
            matching_synonym = candidates[0].lemma

    # If any of the synonyms are single-sense (or nearly single-sense),
    #  then we assume that that is the correct sense
    if not match:
        candidates = []
        for syn in synonyms:
            syn_senses = tdb.ranked_search(lemma=syn,
                                           wordclass=sense.wordclass,
                                           current_only=True)
            if (syn_senses and
                    (tdb.distinct_senses(syn_senses) == 1 or
                    (tdb.distinct_senses(syn_senses) <= 3 and
                    len(synonyms) == 1))):
                candidates.append(syn_senses[0])
        for c in candidates:
            if c.thesclass is not None:
                match = c.thesclass
                matching_synonym = c.lemma
                break

    # If the sense can be restricted by Bayes classification(s), try to
    #   find a match for *any* synonym (even if there's only one)
    if not match and synonyms and sense.bayes.is_usable():
        candidates = []
        for syn in synonyms:
            candidates.extend(tdb.ranked_search(lemma=syn,
                                                wordclass=sense.wordclass,
                                                branches=sense.bayes.ids(),
                                                current_only=True))
        if candidates and candidates[0].thesclass is not None:
            match = candidates[0].thesclass
            matching_synonym = candidates[0].lemma

    return match, matching_synonym
    def find_branch_from_superordinate(self, sense):
        """Classify by finding the main or only sense of the superordinate
        """
        if (sense.wordclass not in ('NN', 'JJ') or
                not sense.superordinate or
                len(sense.superordinate) < 3 or
                sense.superordinate in GENERICS):
            return None

        target_sense = None

        # If the superordinate is (more or less) single-sense, we assume that
        #  sense to be the correct one
        candidates = tdb.ranked_search(
            lemma=sense.superordinate,
            wordclass='NN',
            current_only=True)
        if candidates and tdb.distinct_senses(candidates) <= 2:
            target_sense = candidates[0]

        # Otherwise, narrow by Bayes classification
        if target_sense is None and sense.bayes.confidence() >= 8:
            target_sense = tdb.highest_ranked(
                lemma=sense.superordinate,
                wordclass='NN',
                branches=sense.bayes_based_classifications,
                current_only=True)

        # Otherwise, narrow by branches based on subject labels
        if target_sense is None and sense.label_based_classifications:
            target_sense = tdb.highest_ranked(
                lemma=sense.superordinate,
                wordclass='NN',
                branches=sense.label_based_classifications,
                current_only=True)

        # Otherwise, narrow by branches based on cross-references
        if target_sense is None and sense.xref_branches:
            target_sense = tdb.highest_ranked(
                lemma=sense.superordinate,
                wordclass='NN',
                branches=sense.xref_branches,
                current_only=True)

        # Last gasp: If the gloss consists more or less *only* of the
        #   superordinate (e.g. 'an abbey'), then it should be adequate to
        #   just use the main sense of the superordinate, even if it's
        #   multi-sense.
        # But don't risk this is there are cross-references or subject
        #   labels which might suggest a more specific use
        if (target_sense is None and not sense.subjects and
            not sense.xref_branches and sense.gloss is not None):
            g = re.sub(r'^(a|an|the) ', '', sense.gloss.lower())
            if g == sense.superordinate:
                target_sense = MAIN_SENSE_FINDER.main_sense(
                    lemma=sense.superordinate, wordclass='NN')

        # Otherwise, narrow by Bayes classification
        if target_sense is None and sense.bayes.is_usable():
            target_sense = tdb.highest_ranked(
                lemma=sense.superordinate,
                wordclass='NN',
                branches=sense.bayes_based_classifications,
                current_only=True)

        if target_sense is not None and target_sense.thesclass is not None:
            match = target_sense.thesclass
            if sense.wordclass == 'JJ':
                match = tdb.equivalent_class(match, 'JJ')
            return match
        else:
            return None