def test_similarity(self):
self.assertEqual(
metrics.levenshtein_similarity("night", "nacht"),
metrics.similarity("night", "nacht", metrics.LEVENSHTEIN))
self.assertEqual(
metrics.dice_coefficient("night", "nacht"),
metrics.similarity("night", "nacht", metrics.DICE))
def test_similarity(self):
self.assertEqual(
metrics.levenshtein_similarity("night", "nacht"),
metrics.similarity("night", "nacht", metrics.LEVENSHTEIN))
self.assertEqual(metrics.dice_coefficient("night", "nacht"),
metrics.similarity("night", "nacht", metrics.DICE))
print("pattern.metrics.similarity()")
def fuzzySearch(self, result, query):
best, best_i = 0, None
for i in range(len(result) - len(query) + 1):
score = similarity(result[i:i+len(query)], query)
if best < score:
best = score
best_i = i
return result[best_i+len(query):] if best_i != None else ''
def fuzzySearch(self, result, query):
best, best_i = 0, None
for i in range(len(result) - len(query) + 1):
score = similarity(result[i:i + len(query)], query)
if best < score:
best = score
best_i = i
return result[best_i + len(query):] if best_i != None else ''
def joinSimilar(self, t1, t2):
s1, w1 = t1
s2, w2 = t2
if w1 == 0 or w2 == 0: return #already previously merged
sim = similarity(' '.join(s1), ' '.join(s2))
if sim > .75 and sim != 1:
if w1 > w2:
self._grams[s2] = 0
self._grams[s1] += w2
else:
self._grams[s1] = 0
self._grams[s2] += w1
# 1) Suffix "-bie" and prefix "brie-" are almost identical and go together nicely.
# 2) Semantically, "briefing" refers to a one-sided kind of meeting,
# where a meeting is a form of communication that many people find mindless;
# whereas "zombies" are mindless and can't communicate.
# There is a vaguely humoristic connection between the two concepts.
# How about: "the drill sergeant zombriefed the men" ?
# To simulate (2) algorithmically, we'd need lots of learning material.
# Let's see if we can simulate (1) with a few tricks.
from pattern.metrics import similarity
# The similarity() function computes (1 - the Levenshtein distance):
# http://www.clips.ua.ac.be/pages/pattern-metrics#similarity
# The higher the number (0.0-1.0), the more similar two strings are.
print similarity("bie", "brie") # 0.75
# So, given a word (e.g., "briefing"), we could look for a second word
# that we can glue to the left of it - if the prefix of the given word
# and the suffix of the second word are sufficiently similar
# (e.g., similarity >= 0.75) ...
from pattern.en import lexicon # English {word: word type}-dictionary
w1 = "briefing"
for w2 in lexicon.keys():
if w2[0].isupper(): # Exclude proper names like "Herbie".
continue
if len(w2) <= 3: # Length of "zombie" > 3, OK.
continue
# 1) Suffix "-bie" and prefix "brie-" are almost identical and go together nicely.
# 2) Semantically, "briefing" refers to a one-sided kind of meeting,
# where a meeting is a form of communication that many people find mindless;
# whereas "zombies" are mindless and can't communicate.
# There is a vaguely humoristic connection between the two concepts.
# How about: "the drill sergeant zombriefed the men" ?
# To simulate (2) algorithmically, we'd need lots of learning material.
# Let's see if we can simulate (1) with a few tricks.
from pattern.metrics import similarity
# The similarity() function computes (1 - the Levenshtein distance):
# http://www.clips.ua.ac.be/pages/pattern-metrics#similarity
# The higher the number (0.0-1.0), the more similar two strings are.
print similarity("bie", "brie") # 0.75
# So, given a word (e.g., "briefing"), we could look for a second word
# that we can glue to the left of it - if the prefix of the given word
# and the suffix of the second word are sufficiently similar
# (e.g., similarity >= 0.75) ...
from pattern.en import lexicon # English {word: word type}-dictionary
w1 = "briefing"
for w2 in lexicon.keys():
if w2[0].isupper(): # Exclude proper names like "Herbie".
continue
if len(w2) <= 3: # Length of "zombie" > 3, OK.
continue
