def needleman_wunsch_score(s1, s2, match=2, mismatch=-1, gap=-0.5, score_table={}): utils.check_for_none(s1, s2) utils.check_for_type(basestring, s1, s2) s1 = utils.unicode_normalize(s1) s2 = utils.unicode_normalize(s2) n1, n2 = len(s1), len(s2) if n1 == 0 and n2 == 0: return 0 # construct matrix to get max score of all possible alignments dp = [[0] * (n2 + 1) for _ in range(n1 + 1)] for i in xrange(n1 + 1): for j in xrange(n2 + 1): if i == 0 and j == 0: # [0,0] continue elif i == 0: # most top row dp[i][j] = gap + dp[i][j - 1] elif j == 0: # most left column dp[i][j] = gap + dp[i - 1][j] else: dp[i][j] = max( dp[i][j - 1] + gap, dp[i - 1][j] + gap, dp[i - 1][j - 1] + _get_score( s1[i - 1], s2[j - 1], match, mismatch, score_table)) return dp[n1][n2]
def damerau_levenshtein_distance(s1, s2): """ Similar to Levenshtein, Damerau-Levenshtein distance is the minimum number of operations needed to transform one string into the other, where an operation is defined as an insertion, deletion, or substitution of a single character, or a transposition of two adjacent characters. Args: s1 (str): Sequence 1. s2 (str): Sequence 2. Returns: float: Damerau Levenshtein Distance. Examples: >>> rltk.damerau_levenshtein_distance('abcd', 'acbd') 1 >>> rltk.damerau_levenshtein_distance('abbd', 'acad') 2 """ utils.check_for_none(s1, s2) utils.check_for_type(basestring, s1, s2) s1 = utils.unicode_normalize(s1) s2 = utils.unicode_normalize(s2) n1, n2 = len(s1), len(s2) infinite = n1 + n2 char_arr = defaultdict(int) dp = [[0] * (n2 + 2) for _ in xrange(n1 + 2)] dp[0][0] = infinite for i in xrange(0, n1 + 1): dp[i + 1][0] = infinite dp[i + 1][1] = i for i in xrange(0, n2 + 1): dp[0][i + 1] = infinite dp[1][i + 1] = i for i in xrange(1, n1 + 1): db = 0 for j in xrange(1, n2 + 1): i1 = char_arr[s2[j - 1]] j1 = db cost = 1 if s1[i - 1] == s2[j - 1]: cost = 0 db = j dp[i + 1][j + 1] = min( dp[i][j] + cost, dp[i + 1][j] + 1, dp[i][j + 1] + 1, dp[i1][j1] + (i - i1 - 1) + 1 + (j - j1 - 1)) char_arr[s1[i - 1]] = i return dp[n1 + 1][n2 + 1]
def hamming_distance(s1, s2): utils.check_for_none(s1, s2) # utils.check_for_type(basestring, s1, s2) if type(s1) != type(s2): raise TypeError('Different type') if isinstance(s1, basestring) and isinstance(s2, basestring): s1 = utils.unicode_normalize(s1) s2 = utils.unicode_normalize(s2) if len(s1) != len(s2): raise ValueError('Unequal length') return sum(c1 != c2 for c1, c2 in zip(s1, s2))
def _soundex(s): """ The standard used for this implementation is provided by `U.S. Census Bureau <https://www.archives.gov/research/census/soundex.html>`_. Args: s (str): Sequence. Returns: str: Coded sequence. Examples: >>> rltk.soundex('ashcraft') 'A261' >>> rltk.soundex('pineapple') 'P514' """ utils.check_for_none(s) utils.check_for_type(basestring, s) s = utils.unicode_normalize(s) if len(s) == 0: raise ValueError('Empty string') s = s.upper() CODES = ( ('BFPV', '1'), ('CGJKQSXZ', '2'), ('DT', '3'), ('L', '4'), ('MN', '5'), ('R', '6'), ('AEIOUHWY', '.') # placeholder ) CODE_DICT = dict((c, replace) for chars, replace in CODES for c in chars) sdx = s[0] for i in xrange(1, len(s)): if s[i] not in CODE_DICT: continue code = CODE_DICT[s[i]] if code == '.': continue if s[i] == s[i - 1]: # ignore same letter continue if s[i - 1] in CODE_DICT and CODE_DICT[s[ i - 1]] == code: # 'side-by-side' rule continue if s[i-1] in ('H', 'W') and i - 2 > 0 and\ s[i-2] in CODE_DICT and CODE_DICT[s[i-2]] != '.': # consonant separators continue sdx += code sdx = sdx[0:4].ljust(4, '0') return sdx
def _jaro_distance(s1, s2): # code from https://github.com/nap/jaro-winkler-distance # Copyright Jean-Bernard Ratte utils.check_for_none(s1, s2) utils.check_for_type(basestring, s1, s2) s1 = utils.unicode_normalize(s1) s2 = utils.unicode_normalize(s2) shorter, longer = s1.lower(), s2.lower() if len(s1) > len(s2): longer, shorter = shorter, longer m1 = _get_matching_characters(shorter, longer) m2 = _get_matching_characters(longer, shorter) if len(m1) == 0 or len(m2) == 0: return 0.0 return (float(len(m1)) / len(shorter) + float(len(m2)) / len(longer) + float(len(m1) - _transpositions(m1, m2)) / len(m1)) / 3.0
def _nysiis(s): """ New York State Immunization Information System (NYSIIS) Phonetic Code is a phonetic algorithm created by `The New York State Department of Health's (NYSDOH) Bureau of Immunization <https://www.health.ny.gov/prevention/immunization/information_system/>`_. Args: s (str): Sequence. Returns: str: Coded sequence. Examples: >>> rltk.metaphone('ashcraft') 'AXKRFT' >>> rltk.metaphone('pineapple') 'PNPL' """ # code from https://github.com/jamesturk/jellyfish # Copyright (c) 2015, James Turk # Copyright (c) 2015, Sunlight Foundation # All rights reserved. utils.check_for_none(s) utils.check_for_type(basestring, s) s = utils.unicode_normalize(s) if len(s) == 0: raise ValueError('Empty string') s = s.upper() key = [] # step 1 - prefixes if s.startswith('MAC'): s = 'MCC' + s[3:] elif s.startswith('KN'): s = s[1:] elif s.startswith('K'): s = 'C' + s[1:] elif s.startswith(('PH', 'PF')): s = 'FF' + s[2:] elif s.startswith('SCH'): s = 'SSS' + s[3:] # step 2 - suffixes if s.endswith(('IE', 'EE')): s = s[:-2] + 'Y' elif s.endswith(('DT', 'RT', 'RD', 'NT', 'ND')): s = s[:-2] + 'D' # step 3 - first character of key comes from name key.append(s[0]) # step 4 - translate remaining chars i = 1 len_s = len(s) while i < len_s: ch = s[i] if ch == 'E' and i + 1 < len_s and s[i + 1] == 'V': ch = 'AF' i += 1 elif ch in 'AEIOU': ch = 'A' elif ch == 'Q': ch = 'G' elif ch == 'Z': ch = 'S' elif ch == 'M': ch = 'N' elif ch == 'K': if i + 1 < len(s) and s[i + 1] == 'N': ch = 'N' else: ch = 'C' elif ch == 'S' and s[i + 1:i + 3] == 'CH': ch = 'SS' i += 2 elif ch == 'P' and i + 1 < len(s) and s[i + 1] == 'H': ch = 'F' i += 1 elif ch == 'H' and (s[i - 1] not in 'AEIOU' or (i + 1 < len(s) and s[i + 1] not in 'AEIOU')): if s[i - 1] in 'AEIOU': ch = 'A' else: ch = s[i - 1] elif ch == 'W' and s[i - 1] in 'AEIOU': ch = s[i - 1] if ch[-1] != key[-1][-1]: key.append(ch) i += 1 key = ''.join(key) # step 5 - remove trailing S if key.endswith('S') and key != 'S': key = key[:-1] # step 6 - replace AY w/ Y if key.endswith('AY'): key = key[:-2] + 'Y' # step 7 - remove trailing A if key.endswith('A') and key != 'A': key = key[:-1] # step 8 was already done return key
def _metaphone(s): """ Metaphone fundamentally improves on the Soundex algorithm by using information about variations and inconsistencies in English spelling and pronunciation to produce a more accurate encoding, which does a better job of matching words and names which sound similar. As with Soundex, similar-sounding words should share the same keys. Metaphone is available as a built-in operator in a number of systems. Args: s (str): Sequence. Returns: str: Coded sequence. Examples: >>> rltk.metaphone('ashcraft') 'AXKRFT' >>> rltk.metaphone('pineapple') 'PNPL' """ # code from https://github.com/jamesturk/jellyfish # Copyright (c) 2015, James Turk # Copyright (c) 2015, Sunlight Foundation # All rights reserved. utils.check_for_none(s) utils.check_for_type(basestring, s) s = utils.unicode_normalize(s) if len(s) == 0: raise ValueError('Empty string') s = s.lower() result = [] # skip first character if s starts with these if s.startswith(('kn', 'gn', 'pn', 'ac', 'wr', 'ae')): s = s[1:] i = 0 while i < len(s): c = s[i] next = s[i+1] if i < len(s)-1 else '*****' nextnext = s[i+2] if i < len(s)-2 else '*****' # skip doubles except for cc if c == next and c != 'c': i += 1 continue if c in 'aeiou': if i == 0 or s[i-1] == ' ': result.append(c) elif c == 'b': if (not (i != 0 and s[i-1] == 'm')) or next: result.append('b') elif c == 'c': if next == 'i' and nextnext == 'a' or next == 'h': result.append('x') i += 1 elif next in 'iey': result.append('s') i += 1 else: result.append('k') elif c == 'd': if next == 'g' and nextnext in 'iey': result.append('j') i += 2 else: result.append('t') elif c in 'fjlmnr': result.append(c) elif c == 'g': if next in 'iey': result.append('j') elif next not in 'hn': result.append('k') elif next == 'h' and nextnext and nextnext not in 'aeiou': i += 1 elif c == 'h': if i == 0 or next in 'aeiou' or s[i-1] not in 'aeiou': result.append('h') elif c == 'k': if i == 0 or s[i-1] != 'c': result.append('k') elif c == 'p': if next == 'h': result.append('f') i += 1 else: result.append('p') elif c == 'q': result.append('k') elif c == 's': if next == 'h': result.append('x') i += 1 elif next == 'i' and nextnext in 'oa': result.append('x') i += 2 else: result.append('s') elif c == 't': if next == 'i' and nextnext in 'oa': result.append('x') elif next == 'h': result.append('0') i += 1 elif next != 'c' or nextnext != 'h': result.append('t') elif c == 'v': result.append('f') elif c == 'w': if i == 0 and next == 'h': i += 1 if nextnext in 'aeiou' or nextnext == '*****': result.append('w') elif c == 'x': if i == 0: if next == 'h' or (next == 'i' and nextnext in 'oa'): result.append('x') else: result.append('s') else: result.append('k') result.append('s') elif c == 'y': if next in 'aeiou': result.append('y') elif c == 'z': result.append('s') elif c == ' ': if len(result) > 0 and result[-1] != ' ': result.append(' ') i += 1 return ''.join(result).upper()
def levenshtein_distance(s1, s2, insert={}, delete={}, substitute={}, insert_default=1, delete_default=1, substitute_default=1): """ The Levenshtein distance between two words is the minimum number of single-character edits (insertions, deletions or substitutions) required to change one word into the other. Args: s1 (str): Sequence 1. s2 (str): Sequence 2. insert (dict(str, int), optional): Insert cost of characters. Defaults to empty dict. delete (dict(str, int), optional): Delete cost of characters. Defaults to empty dict. substitute (dict(str, dict(str, int)), optional): Substitute cost of characters. Defaults to empty dict. insert_default (int, optional): Default value of insert cost. Defaults to 1. delete_default (int, optional): Default value of delete cost. Defaults to 1. substitute_default (int, optional): Default value of substitute cost. Defaults to 1. Returns: int: Levenshtein Distance. Examples: >>> rltk.levenshtein_distance('ab', 'abc') 1 >>> rltk.levenshtein_distance('a', 'abc', insert = {'c':50}, ... insert_default=100, delete_default=100, substitute_default=100) 150 """ utils.check_for_none(s1, s2) utils.check_for_type(basestring, s1, s2) s1 = utils.unicode_normalize(s1) s2 = utils.unicode_normalize(s2) n1, n2 = len(s1), len(s2) if n1 == 0 and n2 == 0: return 0 # if n1 == 0 or n2 == 0: # return max(n1, n2) dp = [[0] * (n2 + 1) for _ in range(n1 + 1)] for i in xrange(n1 + 1): for j in xrange(n2 + 1): if i == 0 and j == 0: # [0,0] continue elif i == 0: # most top row c = s2[j - 1] dp[i][j] = insert[c] if c in insert else insert_default dp[i][j] += dp[i][j - 1] elif j == 0: # most left column c = s1[i - 1] dp[i][j] = delete[c] if c in delete else delete_default dp[i][j] += dp[i - 1][j] else: c1, c2 = s1[i - 1], s2[j - 1] insert_cost = insert[c2] if c2 in insert else insert_default delete_cost = delete[c1] if c1 in delete else delete_default substitute_cost = substitute[c1][c2] \ if c1 in substitute and c2 in substitute[c1] else substitute_default if c1 == c2: dp[i][j] = dp[i - 1][j - 1] else: dp[i][j] = min(dp[i][j - 1] + insert_cost, dp[i - 1][j] + delete_cost, dp[i - 1][j - 1] + substitute_cost) return dp[n1][n2]