def whitespace(input_string):
"""
Tokenizes input string based on white space.
Args:
input_string (str): Input string
Returns:
Token list (list)
Raises:
TypeError : If the input is not a string
Examples:
>>> whitespace('data science')
['data', 'science']
>>> whitespace('data science')
['data', 'science']
>>> whitespace('data\tscience')
['data', 'science']
"""
utils.tok_check_for_none(input_string)
utils.tok_check_for_string_input(input_string)
return input_string.split()
def delimiter(input_string, delim_str=' '):
"""
Tokenizes input string based on the given delimiter.
Args:
input_string (str): Input string
delim_str (str): Delimiter string
Returns:
Token list (list)
Raises:
TypeError : If the input is not a string
Examples:
>>> delimiter('data science')
['data', 'science']
>>> delimiter('data$#$science', '$#$')
['data', 'science']
>>> delimiter('data science', ',')
['data science']
"""
utils.tok_check_for_none(input_string)
utils.tok_check_for_string_input(input_string)
return input_string.split(delim_str)
def jaro_winkler(string1, string2, prefix_weight=0.1):
"""
Computes the Jaro-Winkler measure between two strings.
The Jaro-Winkler measure is designed to capture cases where two strings have a low Jaro score, but share a prefix
and thus are likely to match.
Args:
string1,string2 (str): Input strings
prefix_weight (float): Weight to give the prefix (defaults to 0.1)
Returns:
Jaro-Winkler measure (float)
Raises:
TypeError : If the inputs are not strings or if one of the inputs is None.
Examples:
>>> jaro_winkler('MARTHA', 'MARHTA')
0.9611111111111111
>>> jaro_winkler('DWAYNE', 'DUANE')
0.84
>>> jaro_winkler('DIXON', 'DICKSONX')
0.8133333333333332
"""
# input validations
utils.sim_check_for_none(string1, string2)
utils.tok_check_for_string_input(string1, string2)
# if one of the strings is empty return 0
if utils.sim_check_for_empty(string1, string2):
return 0
jw_score = jaro(string1, string2)
min_len = min(len(string1), len(string2))
# prefix length can be at max 4
j = min(min_len, 4)
i = 0
while i < j and string1[i] == string2[i] and string1[i]:
i += 1
if i:
jw_score += i * prefix_weight * (1 - jw_score)
return jw_score
def qgram(input_string, qval=2):
"""
Tokenizes input string into q-grams.
A q-gram is defined as all sequences of q characters. Q-grams are also known as n-grams and
k-grams.
Args:
input_string (str): Input string
qval (int): Q-gram length (defaults to 2)
Returns:
Token list (list)
Raises:
TypeError : If the input is not a string
Examples:
>>> qgram('database')
['da','at','ta','ab','ba','as','se']
>>> qgram('a')
[]
>>> qgram('database', 3)
['dat', 'ata', 'tab', 'aba', 'bas', 'ase']
"""
utils.tok_check_for_none(input_string)
utils.tok_check_for_string_input(input_string)
qgram_list = []
if len(input_string) < qval or qval < 1:
return qgram_list
qgram_list = [input_string[i:i + qval] for i in _range(len(input_string) - (qval - 1))]
return qgram_list
def hamming_distance(string1, string2):
"""
Computes the Hamming distance between two strings.
The Hamming distance between two strings of equal length is the number of positions at which the corresponding
symbols are different. In another way, it measures the minimum number of substitutions required to change
one string into the other, or the minimum number of errors that could have transformed one string into the other.
Args:
string1,string2 (str): Input strings
Returns:
Hamming distance (int)
Raises:
TypeError : If the inputs are not strings or if one of the inputs is None.
ValueError : If the input strings are not of same length
Examples:
>>> hamming_distance('', '')
0
>>> hamming_distance('alex', 'john')
4
>>> hamming_distance(' ', 'a')
0
>>> hamming_distance('JOHN', 'john')
4
"""
# input validations
utils.sim_check_for_none(string1, string2)
utils.tok_check_for_string_input(string1, string2)
# for Hamming Distance string length should be same
utils.sim_check_for_same_len(string1, string2)
# sum all the mismatch characters at the corresponding index of
# input strings
return sum(bool(ord(c1) - ord(c2)) for c1, c2 in zip(string1, string2))
def jaro(string1, string2):
"""
Computes the Jaro measure between two strings.
The Jaro measure is a type of edit distance, This was developed mainly to compare short strings,
such as first and last names.
Args:
string1,string2 (str): Input strings
Returns:
Jaro measure (float)
Raises:
TypeError : If the inputs are not strings or if one of the inputs is None.
Examples:
>>> jaro('MARTHA', 'MARHTA')
0.9444444444444445
>>> jaro('DWAYNE', 'DUANE')
0.8222222222222223
>>> jaro('DIXON', 'DICKSONX')
0.7666666666666666
"""
# input validations
utils.sim_check_for_none(string1, string2)
utils.tok_check_for_string_input(string1, string2)
# if one of the strings is empty return 0
if utils.sim_check_for_empty(string1, string2):
return 0
len_s1 = len(string1)
len_s2 = len(string2)
max_len = max(len_s1, len_s2)
search_range = (max_len // 2) - 1
if search_range < 0:
search_range = 0
flags_s1 = [False] * len_s1
flags_s2 = [False] * len_s2
common_chars = 0
for i, ch_s1 in enumerate(string1):
low = i - search_range if i > search_range else 0
hi = i + search_range if i + search_range < len_s2 else len_s2 - 1
for j in _range(low, hi + 1):
if not flags_s2[j] and string2[j] == ch_s1:
flags_s1[i] = flags_s2[j] = True
common_chars += 1
break
if not common_chars:
return 0
k = trans_count = 0
for i, f_s1 in enumerate(flags_s1):
if f_s1:
for j in _range(k, len_s2):
if flags_s2[j]:
k = j + 1
break
if string1[i] != string2[j]:
trans_count += 1
trans_count /= 2
common_chars = float(common_chars)
weight = ((common_chars / len_s1 + common_chars / len_s2 +
(common_chars - trans_count) / common_chars)) / 3
return weight
def affine(string1, string2, gap_start=1, gap_continuation=0.5, sim_score=sim_ident):
"""
Computes the Affine gap score between two strings.
The Affine gap measure is an extension of the Needleman-Wunsch measure that handles the longer gaps more
gracefully.
For more information refer to string matching chapter in the DI book.
Args:
string1,string2 (str) : Input strings
gap_start (float): Cost for the gap at the start (defaults to 1)
gap_continuation (float) : Cost for the gap continuation (defaults to 0.5)
sim_score (function) : Function computing similarity score between two chars, represented as strings
(defaults to identity).
Returns:
Affine gap score (float)
Raises:
TypeError : If the inputs are not strings or if one of the inputs is None.
Examples:
>>> affine('dva', 'deeva')
1.5
>>> affine('dva', 'deeve', gap_start=2, gap_continuation=0.5)
-0.5
>>> affine('AAAGAATTCA', 'AAATCA', gap_continuation=0.2, sim_score=lambda s1, s2: (int(1 if s1 == s2 else 0)))
4.4
"""
# input validations
utils.sim_check_for_none(string1, string2)
utils.tok_check_for_string_input(string1, string2)
# if one of the strings is empty return 0
if utils.sim_check_for_empty(string1, string2):
return 0
gap_start = -gap_start
gap_continuation = -gap_continuation
m = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
x = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
y = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
# DP initialization
for i in _range(1, len(string1) + 1):
m[i][0] = -float("inf")
x[i][0] = gap_start + (i - 1) * gap_continuation
y[i][0] = -float("inf")
# DP initialization
for j in _range(1, len(string2) + 1):
m[0][j] = -float("inf")
x[0][j] = -float("inf")
y[0][j] = gap_start + (j - 1) * gap_continuation
# affine gap calculation using DP
for i in _range(1, len(string1) + 1):
for j in _range(1, len(string2) + 1):
# best score between x_1....x_i and y_1....y_j given that x_i is aligned to y_j
m[i][j] = sim_score(string1[i - 1], string2[j - 1]) + max(m[i - 1][j - 1], x[i - 1][j - 1], y[i - 1][j - 1])
# the best score given that x_i is aligned to a gap
x[i][j] = max(gap_start + m[i - 1][j], gap_continuation + x[i - 1][j])
# the best score given that y_j is aligned to a gap
y[i][j] = max(gap_start + m[i][j - 1], gap_continuation + y[i][j - 1])
return max(m[len(string1)][len(string2)], x[len(string1)][len(string2)], y[len(string1)][len(string2)])
