class JaccardTestCase(unittest.TestCase):
def setUp(self):
self.threshold = 0.3
self.matches_using_cart_prod = sim_match(
table_A, table_B, tokenized_table_A, tokenized_table_B, l_attr,
r_attr, get_jaccard_fn(), self.threshold, ['id'], ['id'])
self.size_filter = SizeFilter(table_A, tokenized_table_A, l_attr, tok)
self.size_filter.build_index()
self.prefix_filter = PrefixFilter(table_A, tokenized_table_A, l_attr,
tok, self.threshold, token_ordering)
self.prefix_filter.build_index()
self.position_filter = PositionFilter(table_A, tokenized_table_A,
l_attr, tok, self.threshold,
token_ordering)
self.position_filter.build_index()
self.suffix_filter = SuffixFilter(table_A, tokenized_table_A, l_attr,
tok, self.threshold, token_ordering)
def test_jaccard_match(self):
# test jaccard with position filter, size filter, suffix filter
matches = jaccard_match(
table_A, table_B, tokenized_table_A, tokenized_table_B, l_attr,
r_attr, self.threshold,
[self.position_filter, self.size_filter, self.suffix_filter],
['id'], ['id'])
self.assertTrue(compare_matches(self.matches_using_cart_prod, matches))
# test jaccard with prefix filter, size filter, suffix filter
matches = jaccard_match(
table_A, table_B, tokenized_table_A, tokenized_table_B, l_attr,
r_attr, self.threshold,
[self.prefix_filter, self.size_filter, self.suffix_filter], ['id'],
['id'])
self.assertTrue(compare_matches(self.matches_using_cart_prod, matches))
def test_filter_pair(self, lstring, rstring, tokenizer, sim_measure_type,
threshold, allow_empty, allow_missing,
expected_output):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing)
actual_output = prefix_filter.filter_pair(lstring, rstring)
assert_equal(actual_output, expected_output)
class PrefixFilterTestCase(unittest.TestCase):
def setUp(self):
self.prefix_filter = PrefixFilter(A, A_tokenized, 'str', tok, 0.8,
token_ordering)
self.prefix_filter.build_index()
def test_apply_filter(self):
# prefix filter satisfies
l_tokens = order_using_token_ordering(['aa', 'bb', 'cd', 'ef', 'fg'],
token_ordering)
r_tokens = order_using_token_ordering(['fg', 'cd', 'aa'],
token_ordering)
self.assertTrue(
self.prefix_filter.apply_filter(l_tokens, r_tokens, len(l_tokens),
len(r_tokens), 0.8))
l_tokens = order_using_token_ordering(['aa', 'bb', 'cd', 'ef', 'fg'],
token_ordering)
r_tokens = order_using_token_ordering(['aa'], token_ordering)
self.assertTrue(
self.prefix_filter.apply_filter(l_tokens, r_tokens, len(l_tokens),
len(r_tokens), 0.8))
# prefix filter doesn't satisfy
l_tokens = order_using_token_ordering(['aa', 'bb', 'cd', 'ef', 'fg'],
token_ordering)
r_tokens = order_using_token_ordering(['fg'], token_ordering)
self.assertFalse(
self.prefix_filter.apply_filter(l_tokens, r_tokens, len(l_tokens),
len(r_tokens), 0.8))
# test empty list of tokens
l_tokens = order_using_token_ordering(['aa', 'bb', 'cd', 'ef', 'fg'],
token_ordering)
r_tokens = order_using_token_ordering([], token_ordering)
self.assertFalse(
self.prefix_filter.apply_filter(l_tokens, r_tokens, len(l_tokens),
len(r_tokens), 0.8))
self.assertFalse(
self.prefix_filter.apply_filter(r_tokens, l_tokens, len(r_tokens),
len(l_tokens), 0.8))
def test_find_candidates(self):
# test default case (presence of candidates)
tokens = order_using_token_ordering(['aa', 'ef', 'lp'], token_ordering)
self.assertSetEqual(
self.prefix_filter.find_candidates(tokens, len(tokens), 0.8),
set([0, 3]))
# test empty set of candidates
tokens = order_using_token_ordering(['op', 'lp', 'mp'], token_ordering)
self.assertSetEqual(
self.prefix_filter.find_candidates(tokens, len(tokens), 0.8),
set())
# test empty list of probe tokens
tokens = order_using_token_ordering([], token_ordering)
self.assertSetEqual(
self.prefix_filter.find_candidates(tokens, len(tokens), 0.8),
set())
def setUp(self):
self.threshold = 0.3
self.matches_using_cart_prod = sim_match(
table_A, table_B, tokenized_table_A, tokenized_table_B, l_attr,
r_attr, get_jaccard_fn(), self.threshold, ['id'], ['id'])
self.size_filter = SizeFilter(table_A, tokenized_table_A, l_attr, tok)
self.size_filter.build_index()
self.prefix_filter = PrefixFilter(table_A, tokenized_table_A, l_attr,
tok, self.threshold, token_ordering)
self.prefix_filter.build_index()
self.position_filter = PositionFilter(table_A, tokenized_table_A,
l_attr, tok, self.threshold,
token_ordering)
self.position_filter.build_index()
self.suffix_filter = SuffixFilter(table_A, tokenized_table_A, l_attr,
tok, self.threshold, token_ordering)
def test_filter_tables(self, tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing, args, expected_pairs):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing)
actual_candset = prefix_filter.filter_tables(*args)
expected_output_attrs = ['_id']
l_out_prefix = self.default_l_out_prefix
r_out_prefix = self.default_r_out_prefix
# Check for l_out_prefix in args.
if len(args) > 8:
l_out_prefix = args[8]
expected_output_attrs.append(l_out_prefix + args[2])
# Check for r_out_prefix in args.
if len(args) > 9:
r_out_prefix = args[9]
expected_output_attrs.append(r_out_prefix + args[3])
# Check for l_out_attrs in args.
if len(args) > 6:
if args[6]:
l_out_attrs = remove_redundant_attrs(args[6], args[2])
for attr in l_out_attrs:
expected_output_attrs.append(l_out_prefix + attr)
# Check for r_out_attrs in args.
if len(args) > 7:
if args[7]:
r_out_attrs = remove_redundant_attrs(args[7], args[3])
for attr in r_out_attrs:
expected_output_attrs.append(r_out_prefix + attr)
# verify whether the output table has the necessary attributes.
assert_list_equal(list(actual_candset.columns.values),
expected_output_attrs)
actual_pairs = set()
for idx, row in actual_candset.iterrows():
actual_pairs.add(','.join((str(row[l_out_prefix + args[2]]),
str(row[r_out_prefix + args[3]]))))
# verify whether the actual pairs and the expected pairs match.
assert_equal(len(expected_pairs), len(actual_pairs))
common_pairs = actual_pairs.intersection(expected_pairs)
assert_equal(len(common_pairs), len(expected_pairs))
def test_filter_tables(self, tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing, args, expected_pairs):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing)
actual_candset = prefix_filter.filter_tables(*args)
expected_output_attrs = ['_id']
l_out_prefix = self.default_l_out_prefix
r_out_prefix = self.default_r_out_prefix
# Check for l_out_prefix in args.
if len(args) > 8:
l_out_prefix = args[8]
expected_output_attrs.append(l_out_prefix + args[2])
# Check for r_out_prefix in args.
if len(args) > 9:
r_out_prefix = args[9]
expected_output_attrs.append(r_out_prefix + args[3])
# Check for l_out_attrs in args.
if len(args) > 6:
if args[6]:
l_out_attrs = remove_redundant_attrs(args[6], args[2])
for attr in l_out_attrs:
expected_output_attrs.append(l_out_prefix + attr)
# Check for r_out_attrs in args.
if len(args) > 7:
if args[7]:
r_out_attrs = remove_redundant_attrs(args[7], args[3])
for attr in r_out_attrs:
expected_output_attrs.append(r_out_prefix + attr)
# verify whether the output table has the necessary attributes.
assert_list_equal(list(actual_candset.columns.values),
expected_output_attrs)
actual_pairs = set()
for idx, row in actual_candset.iterrows():
actual_pairs.add(','.join((str(row[l_out_prefix + args[2]]),
str(row[r_out_prefix + args[3]]))))
# verify whether the actual pairs and the expected pairs match.
assert_equal(len(expected_pairs), len(actual_pairs))
common_pairs = actual_pairs.intersection(expected_pairs)
assert_equal(len(common_pairs), len(expected_pairs))
def test_filter_candset(self, tokenizer, sim_measure_type, threshold, args,
expected_pairs):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold)
actual_output_candset = prefix_filter.filter_candset(*args)
# verify whether the output table has the necessary attributes.
assert_list_equal(list(actual_output_candset.columns.values),
list(args[0].columns.values))
actual_pairs = set()
for idx, row in actual_output_candset.iterrows():
actual_pairs.add(','.join((str(row[args[1]]), str(row[args[2]]))))
# verify whether the actual pairs and the expected pairs match.
assert_equal(len(expected_pairs), len(actual_pairs))
common_pairs = actual_pairs.intersection(expected_pairs)
assert_equal(len(common_pairs), len(expected_pairs))
def test_filter_candset(self, tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing, args, expected_pairs):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing)
actual_output_candset = prefix_filter.filter_candset(*args)
# verify whether the output table has the necessary attributes.
assert_list_equal(list(actual_output_candset.columns.values),
list(args[0].columns.values))
actual_pairs = set()
for idx, row in actual_output_candset.iterrows():
actual_pairs.add(','.join((str(row[args[1]]), str(row[args[2]]))))
# verify whether the actual pairs and the expected pairs match.
assert_equal(len(expected_pairs), len(actual_pairs))
common_pairs = actual_pairs.intersection(expected_pairs)
assert_equal(len(common_pairs), len(expected_pairs))
def _edit_distance_join_split(ltable_list, rtable_list,
l_columns, r_columns,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, threshold, comp_op,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress):
"""Perform edit distance join for a split of ltable and rtable"""
# find column indices of key attr, join attr and output attrs in ltable
l_key_attr_index = l_columns.index(l_key_attr)
l_join_attr_index = l_columns.index(l_join_attr)
l_out_attrs_indices = find_output_attribute_indices(l_columns, l_out_attrs)
# find column indices of key attr, join attr and output attrs in rtable
r_key_attr_index = r_columns.index(r_key_attr)
r_join_attr_index = r_columns.index(r_join_attr)
r_out_attrs_indices = find_output_attribute_indices(r_columns, r_out_attrs)
sim_measure_type = 'EDIT_DISTANCE'
# generate token ordering using tokens in l_join_attr
# and r_join_attr
token_ordering = gen_token_ordering_for_tables(
[ltable_list, rtable_list],
[l_join_attr_index, r_join_attr_index],
tokenizer, sim_measure_type)
# cache l_join_attr lengths
l_join_attr_list = []
for row in ltable_list:
l_join_attr_list.append(len(row[l_join_attr_index]))
# Build prefix index on l_join_attr
prefix_index = PrefixIndex(ltable_list, l_join_attr_index,
tokenizer, sim_measure_type, threshold,
token_ordering)
prefix_index.build(False)
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold)
comp_fn = COMP_OP_MAP[comp_op]
sim_fn = get_sim_function(sim_measure_type)
output_rows = []
has_output_attributes = (l_out_attrs is not None or
r_out_attrs is not None)
if show_progress:
prog_bar = pyprind.ProgBar(len(rtable_list))
for r_row in rtable_list:
r_string = r_row[r_join_attr_index]
r_len = len(r_string)
r_ordered_tokens = order_using_token_ordering(
tokenizer.tokenize(r_string), token_ordering)
# obtain candidates by applying prefix filter.
candidates = prefix_filter.find_candidates(r_ordered_tokens,
prefix_index)
for cand in candidates:
if r_len - threshold <= l_join_attr_list[cand] <= r_len + threshold:
l_row = ltable_list[cand]
# compute the actual edit distance
edit_dist = sim_fn(l_row[l_join_attr_index], r_string)
if comp_fn(edit_dist, threshold):
if has_output_attributes:
output_row = get_output_row_from_tables(
l_row, r_row,
l_key_attr_index, r_key_attr_index,
l_out_attrs_indices,
r_out_attrs_indices)
else:
output_row = [l_row[l_key_attr_index],
r_row[r_key_attr_index]]
# if out_sim_score flag is set, append the edit distance
# score to the output record.
if out_sim_score:
output_row.append(edit_dist)
output_rows.append(output_row)
if show_progress:
prog_bar.update()
output_header = get_output_header_from_tables(
l_key_attr, r_key_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix)
if out_sim_score:
output_header.append("_sim_score")
# generate a dataframe from the list of output rows
output_table = pd.DataFrame(output_rows, columns=output_header)
return output_table
def test_invalid_threshold(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
1.2)
def test_invalid_sim_measure_type(self):
prefix_filter = PrefixFilter(self.tokenizer, 'INVALID_TYPE',
self.threshold)
def test_invalid_tokenizer_for_edit_distance(self):
prefix_filter = PrefixFilter(self.tokenizer, 'EDIT_DISTANCE', 2)
def test_invalid_r_out_attr(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr',
'B.attr', ['A.attr'], ['B.invalid_attr'])
def test_numeric_r_filter_attr(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr',
'B.int_attr')
def test_invalid_rtable(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, [], 'A.id', 'B.id', 'A.attr',
'B.attr')
def _edit_distance_join_split(ltable_list, rtable_list, l_columns, r_columns,
l_key_attr, r_key_attr, l_join_attr, r_join_attr,
tokenizer, threshold, comp_op, l_out_attrs,
r_out_attrs, l_out_prefix, r_out_prefix,
out_sim_score, show_progress):
"""Perform edit distance join for a split of ltable and rtable"""
# find column indices of key attr, join attr and output attrs in ltable
l_key_attr_index = l_columns.index(l_key_attr)
l_join_attr_index = l_columns.index(l_join_attr)
l_out_attrs_indices = find_output_attribute_indices(l_columns, l_out_attrs)
# find column indices of key attr, join attr and output attrs in rtable
r_key_attr_index = r_columns.index(r_key_attr)
r_join_attr_index = r_columns.index(r_join_attr)
r_out_attrs_indices = find_output_attribute_indices(r_columns, r_out_attrs)
sim_measure_type = 'EDIT_DISTANCE'
# generate token ordering using tokens in l_join_attr
# and r_join_attr
token_ordering = gen_token_ordering_for_tables(
[ltable_list, rtable_list], [l_join_attr_index, r_join_attr_index],
tokenizer, sim_measure_type)
# cache l_join_attr lengths
l_join_attr_list = []
for row in ltable_list:
l_join_attr_list.append(len(row[l_join_attr_index]))
# Build prefix index on l_join_attr
prefix_index = PrefixIndex(ltable_list, l_join_attr_index, tokenizer,
sim_measure_type, threshold, token_ordering)
prefix_index.build(False)
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold)
comp_fn = COMP_OP_MAP[comp_op]
sim_fn = get_sim_function(sim_measure_type)
output_rows = []
has_output_attributes = (l_out_attrs is not None
or r_out_attrs is not None)
if show_progress:
prog_bar = pyprind.ProgBar(len(rtable_list))
for r_row in rtable_list:
r_string = r_row[r_join_attr_index]
r_len = len(r_string)
r_ordered_tokens = order_using_token_ordering(
tokenizer.tokenize(r_string), token_ordering)
# obtain candidates by applying prefix filter.
candidates = prefix_filter.find_candidates(r_ordered_tokens,
prefix_index)
for cand in candidates:
if r_len - threshold <= l_join_attr_list[cand] <= r_len + threshold:
l_row = ltable_list[cand]
# compute the actual edit distance
edit_dist = sim_fn(l_row[l_join_attr_index], r_string)
if comp_fn(edit_dist, threshold):
if has_output_attributes:
output_row = get_output_row_from_tables(
l_row, r_row, l_key_attr_index, r_key_attr_index,
l_out_attrs_indices, r_out_attrs_indices)
else:
output_row = [
l_row[l_key_attr_index], r_row[r_key_attr_index]
]
# if out_sim_score flag is set, append the edit distance
# score to the output record.
if out_sim_score:
output_row.append(edit_dist)
output_rows.append(output_row)
if show_progress:
prog_bar.update()
output_header = get_output_header_from_tables(l_key_attr, r_key_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix)
if out_sim_score:
output_header.append("_sim_score")
# generate a dataframe from the list of output rows
output_table = pd.DataFrame(output_rows, columns=output_header)
return output_table
def test_invalid_rtable(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, [], 'A.id', 'B.id',
'A.attr', 'B.attr')
def test_filter_pair(self, lstring, rstring, tokenizer, sim_measure_type,
threshold, allow_empty, allow_missing, expected_output):
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold,
allow_empty, allow_missing)
actual_output = prefix_filter.filter_pair(lstring, rstring)
assert_equal(actual_output, expected_output)
def test_numeric_r_filter_attr(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id',
'A.attr', 'B.int_attr')
def _edit_dist_join_split(ltable, rtable,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer,
threshold,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score):
# find column indices of key attr, join attr and output attrs in ltable
l_columns = list(ltable.columns.values)
l_key_attr_index = l_columns.index(l_key_attr)
l_join_attr_index = l_columns.index(l_join_attr)
l_out_attrs_indices = find_output_attribute_indices(l_columns, l_out_attrs)
# find column indices of key attr, join attr and output attrs in rtable
r_columns = list(rtable.columns.values)
r_key_attr_index = r_columns.index(r_key_attr)
r_join_attr_index = r_columns.index(r_join_attr)
r_out_attrs_indices = find_output_attribute_indices(r_columns, r_out_attrs)
# build a dictionary on ltable
ltable_dict = build_dict_from_table(ltable, l_key_attr_index,
l_join_attr_index)
# build a dictionary on rtable
rtable_dict = build_dict_from_table(rtable, r_key_attr_index,
r_join_attr_index)
sim_measure_type = 'EDIT_DISTANCE'
# generate token ordering using tokens in l_join_attr
# and r_join_attr
token_ordering = gen_token_ordering_for_tables(
[ltable_dict.values(),
rtable_dict.values()],
[l_join_attr_index,
r_join_attr_index],
tokenizer, sim_measure_type)
# build a dictionary of l_join_attr lengths
l_join_attr_dict = {}
for row in ltable_dict.values():
l_join_attr_dict[row[l_key_attr_index]] = len(str(
row[l_join_attr_index]))
# Build prefix index on l_join_attr
prefix_index = PrefixIndex(ltable_dict.values(),
l_key_attr_index, l_join_attr_index,
tokenizer, sim_measure_type, threshold,
token_ordering)
prefix_index.build()
prefix_filter = PrefixFilter(tokenizer, sim_measure_type, threshold)
sim_fn = get_sim_function(sim_measure_type)
output_rows = []
has_output_attributes = (l_out_attrs is not None or
r_out_attrs is not None)
prog_bar = pyprind.ProgBar(len(rtable_dict.keys()))
for r_row in rtable_dict.values():
r_id = r_row[r_key_attr_index]
r_string = str(r_row[r_join_attr_index])
r_len = len(r_string)
# check for empty string
if not r_string:
continue
r_join_attr_tokens = tokenize(r_string, tokenizer, sim_measure_type)
r_ordered_tokens = order_using_token_ordering(r_join_attr_tokens,
token_ordering)
candidates = find_candidates_prefix_filter(
r_ordered_tokens, len(r_ordered_tokens),
prefix_filter, prefix_index)
for cand in candidates:
if r_len - threshold <= l_join_attr_dict[cand] <= r_len + threshold:
edit_dist = sim_fn(str(ltable_dict[cand][l_join_attr_index]),
r_string)
if edit_dist <= threshold:
if has_output_attributes:
output_row = get_output_row_from_tables(
ltable_dict[cand], r_row,
cand, r_id,
l_out_attrs_indices,
r_out_attrs_indices)
if out_sim_score:
output_row.append(edit_dist)
output_rows.append(output_row)
else:
output_row = [cand, r_id]
if out_sim_score:
output_row.append(edit_dist)
output_rows.append(output_row)
prog_bar.update()
output_header = get_output_header_from_tables(
l_key_attr, r_key_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix)
if out_sim_score:
output_header.append("_sim_score")
# generate a dataframe from the list of output rows
output_table = pd.DataFrame(output_rows, columns=output_header)
return output_table
def setUp(self):
self.prefix_filter = PrefixFilter(A, A_tokenized, 'str', tok, 0.8,
token_ordering)
self.prefix_filter.build_index()
def test_invalid_r_out_attr(self):
prefix_filter = PrefixFilter(self.tokenizer, self.sim_measure_type,
self.threshold)
prefix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id',
'A.attr', 'B.attr',
['A.attr'], ['B.invalid_attr'])