def find_in_index_range(start_index):
while True:
try:
first_index = sequence.index(first_item, start_index, first_item_last_index)
start_index = first_index + 1
except ValueError:
return -1
for subseq_index in xrange(1, len(subsequence)):
if sequence[first_index + subseq_index] != subsequence[subseq_index]:
break
else:
return first_index
def find_in_index_range(start_index):
while True:
try:
first_index = sequence.index(first_item, start_index,
first_item_last_index)
start_index = first_index + 1
except ValueError:
return -1
for subseq_index in xrange(1, len(subsequence)):
if sequence[first_index +
subseq_index] != subsequence[subseq_index]:
break
else:
return first_index
def find_near_matches_levenshtein_ngrams(subsequence, sequence, max_l_dist):
subseq_len = len(subsequence)
seq_len = len(sequence)
ngram_len = subseq_len // (max_l_dist + 1)
if ngram_len == 0:
raise ValueError(
'the subsequence length must be greater than max_l_dist')
def make_match(start, end, dist):
return Match(start, end, dist, matched=sequence[start:end])
for ngram_start in xrange(0, subseq_len - ngram_len + 1, ngram_len):
ngram_end = ngram_start + ngram_len
subseq_before_reversed = subsequence[:ngram_start][::-1]
subseq_after = subsequence[ngram_end:]
start_index = max(0, ngram_start - max_l_dist)
end_index = min(seq_len, seq_len - subseq_len + ngram_end + max_l_dist)
for index in search_exact(subsequence[ngram_start:ngram_end], sequence,
start_index, end_index):
# try to expand left and/or right according to n_ngram
dist_right, right_expand_size = _expand(
subseq_after,
sequence[index + ngram_len:index - ngram_start + subseq_len +
max_l_dist],
max_l_dist,
)
if dist_right is None:
continue
dist_left, left_expand_size = _expand(
subseq_before_reversed,
sequence[max(0, index - ngram_start -
(max_l_dist - dist_right)):index][::-1],
max_l_dist - dist_right,
)
if dist_left is None:
continue
assert dist_left + dist_right <= max_l_dist
yield make_match(
start=index - left_expand_size,
end=index + ngram_len + right_expand_size,
dist=dist_left + dist_right,
)
def find_near_matches_generic_ngrams(subsequence, sequence, search_params):
"""search for near-matches of subsequence in sequence
This searches for near-matches, where the nearly-matching parts of the
sequence must meet the following limitations (relative to the subsequence):
* the maximum allowed number of character substitutions
* the maximum allowed number of new characters inserted
* and the maximum allowed number of character deletions
* the total number of substitutions, insertions and deletions
"""
if not subsequence:
raise ValueError('Given subsequence is empty!')
max_l_dist = search_params.max_l_dist
# optimization: prepare some often used things in advance
subseq_len = len(subsequence)
seq_len = len(sequence)
ngram_len = subseq_len // (max_l_dist + 1)
if ngram_len == 0:
raise ValueError(
'the subsequence length must be greater than max_l_dist')
for ngram_start in xrange(0, subseq_len - ngram_len + 1, ngram_len):
ngram_end = ngram_start + ngram_len
start_index = max(0, ngram_start - max_l_dist)
end_index = min(seq_len, seq_len - subseq_len + ngram_end + max_l_dist)
for index in search_exact(subsequence[ngram_start:ngram_end], sequence,
start_index, end_index):
# try to expand left and/or right according to n_ngram
for match in find_near_matches_generic_linear_programming(
subsequence,
sequence[max(0, index - ngram_start - max_l_dist):index -
ngram_start + subseq_len + max_l_dist],
search_params,
):
yield attr.evolve(
match,
start=match.start +
max(0, index - ngram_start - max_l_dist),
end=match.end + max(0, index - ngram_start - max_l_dist),
)
def find_near_matches_levenshtein_ngrams(subsequence, sequence, max_l_dist):
subseq_len = len(subsequence)
seq_len = len(sequence)
ngram_len = subseq_len // (max_l_dist + 1)
if ngram_len == 0:
raise ValueError('the subsequence length must be greater than max_l_dist')
for ngram_start in xrange(0, subseq_len - ngram_len + 1, ngram_len):
ngram_end = ngram_start + ngram_len
subseq_before_reversed = subsequence[:ngram_start][::-1]
subseq_after = subsequence[ngram_end:]
start_index = max(0, ngram_start - max_l_dist)
end_index = min(seq_len, seq_len - subseq_len + ngram_end + max_l_dist)
for index in search_exact(subsequence[ngram_start:ngram_end], sequence, start_index, end_index):
# try to expand left and/or right according to n_ngram
dist_right, right_expand_size = _expand(
subseq_after,
sequence[index + ngram_len:index - ngram_start + subseq_len + max_l_dist],
max_l_dist,
)
if dist_right is None:
continue
dist_left, left_expand_size = _expand(
subseq_before_reversed,
sequence[max(0, index - ngram_start - (max_l_dist - dist_right)):index][::-1],
max_l_dist - dist_right,
)
if dist_left is None:
continue
assert dist_left + dist_right <= max_l_dist
yield Match(
start=index - left_expand_size,
end=index + ngram_len + right_expand_size,
dist=dist_left + dist_right,
)
def find_near_matches_levenshtein_linear_programming(subsequence, sequence,
max_l_dist):
if not subsequence:
raise ValueError('Given subsequence is empty!')
subseq_len = len(subsequence)
def make_match(start, end, dist):
return Match(start, end, dist, matched=sequence[start:end])
if max_l_dist >= subseq_len:
for index in xrange(len(sequence) + 1):
yield make_match(index, index, subseq_len)
return
# optimization: prepare some often used things in advance
char2first_subseq_index = make_char2first_subseq_index(subsequence,
max_l_dist)
candidates = []
for index, char in enumerate(sequence):
new_candidates = []
idx_in_subseq = char2first_subseq_index.get(char, None)
if idx_in_subseq is not None:
if idx_in_subseq + 1 == subseq_len:
yield make_match(index, index + 1, idx_in_subseq)
else:
new_candidates.append(Candidate(index, idx_in_subseq + 1, idx_in_subseq))
for cand in candidates:
# if this sequence char is the candidate's next expected char
if subsequence[cand.subseq_index] == char:
# if reached the end of the subsequence, return a match
if cand.subseq_index + 1 == subseq_len:
yield make_match(cand.start, index + 1, cand.dist)
# otherwise, update the candidate's subseq_index and keep it
else:
new_candidates.append(cand._replace(
subseq_index=cand.subseq_index + 1,
))
# if this sequence char is *not* the candidate's next expected char
else:
# we can try skipping a sequence or sub-sequence char (or both),
# unless this candidate has already skipped the maximum allowed
# number of characters
if cand.dist == max_l_dist:
continue
# add a candidate skipping a sequence char
new_candidates.append(cand._replace(dist=cand.dist + 1))
if index + 1 < len(sequence) and cand.subseq_index + 1 < subseq_len:
# add a candidate skipping both a sequence char and a
# subsequence char
new_candidates.append(cand._replace(
dist=cand.dist + 1,
subseq_index=cand.subseq_index + 1,
))
# try skipping subsequence chars
for n_skipped in xrange(1, max_l_dist - cand.dist + 1):
# if skipping n_skipped sub-sequence chars reaches the end
# of the sub-sequence, yield a match
if cand.subseq_index + n_skipped == subseq_len:
yield make_match(cand.start, index + 1,
cand.dist + n_skipped)
break
# otherwise, if skipping n_skipped sub-sequence chars
# reaches a sub-sequence char identical to this sequence
# char, add a candidate skipping n_skipped sub-sequence
# chars
elif subsequence[cand.subseq_index + n_skipped] == char:
# if this is the last char of the sub-sequence, yield
# a match
if cand.subseq_index + n_skipped + 1 == subseq_len:
yield make_match(cand.start, index + 1,
cand.dist + n_skipped)
# otherwise add a candidate skipping n_skipped
# subsequence chars
else:
new_candidates.append(cand._replace(
dist=cand.dist + n_skipped,
subseq_index=cand.subseq_index + 1 + n_skipped,
))
break
# note: if the above loop ends without a break, that means that
# no candidate could be added / yielded by skipping sub-sequence
# chars
candidates = new_candidates
for cand in candidates:
dist = cand.dist + subseq_len - cand.subseq_index
if dist <= max_l_dist:
yield make_match(cand.start, len(sequence), dist)
def _find_near_matches_generic_linear_programming(subsequence, sequence, search_params):
"""search for near-matches of subsequence in sequence
This searches for near-matches, where the nearly-matching parts of the
sequence must meet the following limitations (relative to the subsequence):
* the maximum allowed number of character substitutions
* the maximum allowed number of new characters inserted
* and the maximum allowed number of character deletions
* the total number of substitutions, insertions and deletions
"""
if not subsequence:
raise ValueError('Given subsequence is empty!')
max_substitutions, max_insertions, max_deletions, max_l_dist = search_params.unpacked
# optimization: prepare some often used things in advance
subseq_len = len(subsequence)
candidates = []
for index, char in enumerate(sequence):
candidates.append(GenericSearchCandidate(index, 0, 0, 0, 0, 0))
new_candidates = []
for cand in candidates:
# if this sequence char is the candidate's next expected char
if char == subsequence[cand.subseq_index]:
# if reached the end of the subsequence, return a match
if cand.subseq_index + 1 == subseq_len:
yield Match(cand.start, index + 1, cand.l_dist)
# otherwise, update the candidate's subseq_index and keep it
else:
new_candidates.append(cand._replace(
subseq_index=cand.subseq_index + 1,
))
# if this sequence char is *not* the candidate's next expected char
else:
# we can try skipping a sequence or sub-sequence char (or both),
# unless this candidate has already skipped the maximum allowed
# number of characters
if cand.l_dist == max_l_dist:
continue
if cand.n_ins < max_insertions:
# add a candidate skipping a sequence char
new_candidates.append(cand._replace(
n_ins=cand.n_ins + 1,
l_dist=cand.l_dist + 1,
))
if cand.subseq_index + 1 < subseq_len:
if cand.n_subs < max_substitutions:
# add a candidate skipping both a sequence char and a
# subsequence char
new_candidates.append(cand._replace(
n_subs=cand.n_subs + 1,
subseq_index=cand.subseq_index + 1,
l_dist=cand.l_dist + 1,
))
elif cand.n_dels < max_deletions and cand.n_ins < max_insertions:
# add a candidate skipping both a sequence char and a
# subsequence char
new_candidates.append(cand._replace(
n_ins=cand.n_ins + 1,
n_dels=cand.n_dels + 1,
subseq_index=cand.subseq_index + 1,
l_dist=cand.l_dist + 1,
))
else:
# cand.subseq_index == _subseq_len - 1
if (
cand.n_subs < max_substitutions or
(
cand.n_dels < max_deletions and
cand.n_ins < max_insertions
)
):
yield Match(cand.start, index + 1, cand.l_dist + 1)
# try skipping subsequence chars
for n_skipped in xrange(1, min(max_deletions - cand.n_dels, max_l_dist - cand.l_dist) + 1):
# if skipping n_dels sub-sequence chars reaches the end
# of the sub-sequence, yield a match
if cand.subseq_index + n_skipped == subseq_len:
yield Match(cand.start, index + 1,
cand.l_dist + n_skipped)
break
# otherwise, if skipping n_skipped sub-sequence chars
# reaches a sub-sequence char identical to this sequence
# char ...
elif subsequence[cand.subseq_index + n_skipped] == char:
# if this is the last char of the sub-sequence, yield
# a match
if cand.subseq_index + n_skipped + 1 == subseq_len:
yield Match(cand.start, index + 1,
cand.l_dist + n_skipped)
# otherwise add a candidate skipping n_skipped
# subsequence chars
else:
new_candidates.append(cand._replace(
n_dels=cand.n_dels + n_skipped,
subseq_index=cand.subseq_index + 1 + n_skipped,
l_dist=cand.l_dist + n_skipped,
))
break
# note: if the above loop ends without a break, that means that
# no candidate could be added / yielded by skipping sub-sequence
# chars
candidates = new_candidates
for cand in candidates:
# note: index + 1 == length(sequence)
n_skipped = subseq_len - cand.subseq_index
if cand.n_dels + n_skipped <= max_deletions and \
cand.l_dist + n_skipped <= max_l_dist:
yield Match(cand.start, index + 1, cand.l_dist + n_skipped)
def find_near_matches_levenshtein_linear_programming(subsequence, sequence,
max_l_dist):
if not subsequence:
raise ValueError('Given subsequence is empty!')
subseq_len = len(subsequence)
if max_l_dist >= subseq_len:
for index in xrange(len(sequence) + 1):
yield Match(index, index, subseq_len)
return
# optimization: prepare some often used things in advance
char2first_subseq_index = make_char2first_subseq_index(subsequence,
max_l_dist)
candidates = []
for index, char in enumerate(sequence):
new_candidates = []
idx_in_subseq = char2first_subseq_index.get(char, None)
if idx_in_subseq is not None:
if idx_in_subseq + 1 == subseq_len:
yield Match(index, index + 1, idx_in_subseq)
else:
new_candidates.append(Candidate(index, idx_in_subseq + 1, idx_in_subseq))
for cand in candidates:
# if this sequence char is the candidate's next expected char
if subsequence[cand.subseq_index] == char:
# if reached the end of the subsequence, return a match
if cand.subseq_index + 1 == subseq_len:
yield Match(cand.start, index + 1, cand.dist)
# otherwise, update the candidate's subseq_index and keep it
else:
new_candidates.append(cand._replace(
subseq_index=cand.subseq_index + 1,
))
# if this sequence char is *not* the candidate's next expected char
else:
# we can try skipping a sequence or sub-sequence char (or both),
# unless this candidate has already skipped the maximum allowed
# number of characters
if cand.dist == max_l_dist:
continue
# add a candidate skipping a sequence char
new_candidates.append(cand._replace(dist=cand.dist + 1))
if index + 1 < len(sequence) and cand.subseq_index + 1 < subseq_len:
# add a candidate skipping both a sequence char and a
# subsequence char
new_candidates.append(cand._replace(
dist=cand.dist + 1,
subseq_index=cand.subseq_index + 1,
))
# try skipping subsequence chars
for n_skipped in xrange(1, max_l_dist - cand.dist + 1):
# if skipping n_skipped sub-sequence chars reaches the end
# of the sub-sequence, yield a match
if cand.subseq_index + n_skipped == subseq_len:
yield Match(cand.start, index + 1, cand.dist + n_skipped)
break
# otherwise, if skipping n_skipped sub-sequence chars
# reaches a sub-sequence char identical to this sequence
# char, add a candidate skipping n_skipped sub-sequence
# chars
elif subsequence[cand.subseq_index + n_skipped] == char:
# if this is the last char of the sub-sequence, yield
# a match
if cand.subseq_index + n_skipped + 1 == subseq_len:
yield Match(cand.start, index + 1,
cand.dist + n_skipped)
# otherwise add a candidate skipping n_skipped
# subsequence chars
else:
new_candidates.append(cand._replace(
dist=cand.dist + n_skipped,
subseq_index=cand.subseq_index + 1 + n_skipped,
))
break
# note: if the above loop ends without a break, that means that
# no candidate could be added / yielded by skipping sub-sequence
# chars
candidates = new_candidates
for cand in candidates:
dist = cand.dist + subseq_len - cand.subseq_index
if dist <= max_l_dist:
yield Match(cand.start, len(sequence), dist)