def signatures(ambiguous_word, pos=None, hyperhypo=True, adapted=False,
remove_stopwords=True, to_lemmatize=True, remove_numbers=True,
lowercase=True, to_stem=False, original_lesk=False, from_cache=True):
"""
:param ambiguous_word: The ambiguous word.
:type ambiguous_word: str
"""
# Ensure that the POS is supported.
pos = pos if pos in ['a', 'r', 's', 'n', 'v', None] else None
# Holds the synset->signature dictionary.
ss_sign = {}
for ss in wn.synsets(ambiguous_word, pos):
ss_sign[ss] = synset_signatures(ss, hyperhypo=hyperhypo,
adapted=adapted,
remove_stopwords=remove_stopwords,
to_lemmatize=to_lemmatize,
remove_numbers=remove_numbers,
lowercase=lowercase,
original_lesk=original_lesk,
from_cache=from_cache)
# Matching exact words may cause sparsity, so optional matching for stems.
# Not advisible to use thus left out of the synsets_signatures()
if to_stem == True:
ss_sign = {ss:[porter.stem(s) for s in signature]
for ss, signature in ss_sign.items()}
return ss_sign
def adapted_lesk(context_sentence, ambiguous_word, \
pos=None, lemma=True, stem=True, hyperhypo=True, \
stop=True, context_is_lemmatized=False, \
nbest=False, keepscore=False, normalizescore=False):
"""
This function is the implementation of the Adapted Lesk algorithm,
described in Banerjee and Pederson (2002). It makes use of the lexical
items from semantically related senses within the wordnet
hierarchies and to generate more lexical items for each sense.
see www.d.umn.edu/~tpederse/Pubs/cicling2002-b.pdf
"""
# Ensure that ambiguous word is a lemma.
ambiguous_word = lemmatize(ambiguous_word)
# If ambiguous word not in WordNet return None
if not wn.synsets(ambiguous_word):
return None
# Get the signatures for each synset.
ss_sign = simple_signature(ambiguous_word, pos, lemma, stem, hyperhypo)
for ss in ss_sign:
# Includes holonyms.
ss_mem_holonyms = synset_properties(ss, 'member_holonyms')
ss_part_holonyms = synset_properties(ss, 'part_holonyms')
ss_sub_holonyms = synset_properties(ss, 'substance_holonyms')
# Includes meronyms.
ss_mem_meronyms = synset_properties(ss, 'member_meronyms')
ss_part_meronyms = synset_properties(ss, 'part_meronyms')
ss_sub_meronyms = synset_properties(ss, 'substance_meronyms')
# Includes similar_tos
ss_simto = synset_properties(ss, 'similar_tos')
related_senses = list(
set(ss_mem_holonyms + ss_part_holonyms + ss_sub_holonyms +
ss_mem_meronyms + ss_part_meronyms + ss_sub_meronyms +
ss_simto))
signature = list([
j for j in chain(
*[synset_properties(i, 'lemma_names') for i in related_senses])
if j not in EN_STOPWORDS
])
# Lemmatized context is preferred over stemmed context
if lemma == True:
signature = [lemmatize(i) for i in signature]
# Matching exact words causes sparsity, so optional matching for stems.
if stem == True:
signature = [porter.stem(i) for i in signature]
# Adds the extended signature to the simple signatures.
ss_sign[ss] += signature
# Disambiguate the sense in context.
if context_is_lemmatized:
context_sentence = context_sentence.split()
else:
context_sentence = lemmatize_sentence(context_sentence)
best_sense = compare_overlaps(context_sentence, ss_sign, \
nbest=nbest, keepscore=keepscore, \
normalizescore=normalizescore)
return best_sense
def simple_signature(ss, stem=False):
"""
Returns a synsets_signatures dictionary that includes signature words of a
sense from its:
(i) definition
(ii) example sentences
(iii) hypernyms and hyponyms
"""
signature = []
# print("ss: ",ss)
# Includes definition.
ss_definition = synset_properties(ss, 'definition')
signature += word_tokenize(ss_definition)
# Includes examples
ss_examples = synset_properties(ss, 'examples')
signature += list(chain(*[i.split() for i in ss_examples]))
# Includes lemma_names.
ss_lemma_names = synset_properties(ss, 'lemma_names')
signature += ss_lemma_names
# Optional: includes lemma_names of hypernyms and hyponyms.
ss_hyponyms = synset_properties(ss, 'hyponyms')
ss_hypernyms = synset_properties(ss, 'hypernyms')
ss_hypohypernyms = ss_hypernyms + ss_hyponyms
signature += list(chain(*[i.lemma_names() for i in ss_hypohypernyms]))
# print(signature)
'''
# Includes holonyms.
ss_mem_holonyms = synset_properties(ss, 'member_holonyms')
ss_part_holonyms = synset_properties(ss, 'part_holonyms')
ss_sub_holonyms = synset_properties(ss, 'substance_holonyms')
# Includes meronyms.
ss_mem_meronyms = synset_properties(ss, 'member_meronyms')
ss_part_meronyms = synset_properties(ss, 'part_meronyms')
ss_sub_meronyms = synset_properties(ss, 'substance_meronyms')
# Includes similar_tos
ss_simto = synset_properties(ss, 'similar_tos')
related_senses = list(set(ss_mem_holonyms+ss_part_holonyms+
ss_sub_holonyms+ss_mem_meronyms+
ss_part_meronyms+ss_sub_meronyms+ ss_simto))
signature += list([j for j in chain(*[synset_properties(i, 'lemma_names')
for i in related_senses])
if j not in EN_STOPWORDS])
# print(signature)
# Optional: removes stopwords.
'''
signature = [i for i in signature if i.lower() not in EN_STOPWORDS]
# Lemmatized context is preferred over stemmed context.
signature = [lemmatize(i) for i in signature]
# Matching exact words may cause sparsity, so optional matching for stems.
if stem == True:
signature = [porter.stem(i) for i in signature]
return signature
def normalizar_ctx(lista_ctx, stop=True, lematizar=True, stem=True):
if stop:
lista_ctx = [
i for i in lista_ctx if i not in stopwords.words('english')
]
if lematizar:
lista_ctx = [lemmatize(i) for i in lista_ctx]
if stem:
lista_ctx = [porter.stem(i) for i in lista_ctx]
return lista_ctx
def signatures(ambiguous_word: str,
pos: str = None,
hyperhypo=True,
adapted=False,
remove_stopwords=True,
to_lemmatize=True,
remove_numbers=True,
lowercase=True,
to_stem=False,
original_lesk=False,
from_cache=True) -> dict:
"""
Takes an ambiguous word and optionally its Part-Of-Speech and returns
a dictionary where keys are the synsets and values are sets of signatures.
:param ambiguous_word: String, a single word.
:param pos: String, one of 'a', 'r', 's', 'n', 'v', or None.
:return: dict(synset:{signatures}).
"""
# Ensure that the POS is supported.
pos = pos if pos in ['a', 'r', 's', 'n', 'v', None] else None
# If the POS specified isn't found but other POS is in wordnet.
if not wn.synsets(ambiguous_word, pos) and wn.synsets(ambiguous_word):
pos = None
# Holds the synset->signature dictionary.
ss_sign = {}
for ss in wn.synsets(ambiguous_word, pos):
ss_sign[ss] = synset_signatures(ss,
hyperhypo=hyperhypo,
adapted=adapted,
remove_stopwords=remove_stopwords,
to_lemmatize=to_lemmatize,
remove_numbers=remove_numbers,
lowercase=lowercase,
original_lesk=original_lesk,
from_cache=from_cache)
# Matching exact words may cause sparsity, so optional matching for stems.
# Not advisible to use thus left out of the synsets_signatures()
if to_stem == True:
ss_sign = {
ss: [porter.stem(s) for s in signature]
for ss, signature in ss_sign.items()
}
return ss_sign
def simple_signature(ambiguous_word, pos=None, lemma=True, stem=False, \
hyperhypo=True, stop=True):
"""
Returns a synsets_signatures dictionary that includes signature words of a
sense from its:
(i) definition
(ii) example sentences
(iii) hypernyms and hyponyms
"""
synsets_signatures = {}
for ss in wn.synsets(ambiguous_word):
try: # If POS is specified.
if pos and str(ss.pos()) != pos:
continue
except:
if pos and str(ss.pos) != pos:
continue
signature = []
# Includes definition.
ss_definition = synset_properties(ss, 'definition')
signature += ss_definition
# Includes examples
ss_examples = synset_properties(ss, 'examples')
signature += list(chain(*[i.split() for i in ss_examples]))
# Includes lemma_names.
ss_lemma_names = synset_properties(ss, 'lemma_names')
signature += ss_lemma_names
# Optional: includes lemma_names of hypernyms and hyponyms.
if hyperhypo == True:
ss_hyponyms = synset_properties(ss, 'hyponyms')
ss_hypernyms = synset_properties(ss, 'hypernyms')
ss_hypohypernyms = ss_hypernyms + ss_hyponyms
signature += list(
chain(*[i.lemma_names() for i in ss_hypohypernyms]))
# Optional: removes stopwords.
if stop == True:
signature = [i for i in signature if i not in EN_STOPWORDS]
# Lemmatized context is preferred over stemmed context.
if lemma == True:
signature = [lemmatize(i) for i in signature]
# Matching exact words may cause sparsity, so optional matching for stems.
if stem == True:
signature = [porter.stem(i) for i in signature]
synsets_signatures[ss] = signature
return synsets_signatures
def cosine_lesk(context_sentence, ambiguous_word, \
pos=None, lemma=True, stem=True, hyperhypo=True, \
stop=True, context_is_lemmatized=False, \
nbest=False):
"""
In line with vector space models, we can use cosine to calculate overlaps
instead of using raw overlap counts. Essentially, the idea of using
signatures (aka 'sense paraphrases') is lesk-like.
"""
# Ensure that ambiguous word is a lemma.
ambiguous_word = lemmatize(ambiguous_word)
# If ambiguous word not in WordNet return None
if not wn.synsets(ambiguous_word):
return None
synsets_signatures = simple_signature(ambiguous_word, pos, lemma, stem,
hyperhypo)
if context_is_lemmatized:
context_sentence = " ".join(context_sentence.split())
else:
context_sentence = " ".join(lemmatize_sentence(context_sentence))
scores = []
for ss, signature in synsets_signatures.items():
# Lowercase and replace "_" with spaces.
signature = " ".join(map(str, signature)).lower().replace("_", " ")
# Removes punctuation.
signature = [i for i in word_tokenize(signature) \
if i not in string.punctuation]
# Optional: remove stopwords.
if stop:
signature = [i for i in signature if i not in EN_STOPWORDS]
# Optional: Lemmatize the tokens.
if lemma == True:
signature = [lemmatize(i) for i in signature]
# Optional: stem the tokens.
if stem:
signature = [porter.stem(i) for i in signature]
scores.append((cos_sim(context_sentence, " ".join(signature)), ss))
if not nbest:
return sorted(scores, reverse=True)[0][1]
else:
return [(j, i) for i, j in sorted(scores, reverse=True)]