def test_filter_tables(self, tokenizer, sim_measure_type, threshold, args, expected_pairs): suffix_filter = SuffixFilter(tokenizer, sim_measure_type, threshold) actual_candset = suffix_filter.filter_tables(*args) expected_output_attrs = ['_id'] l_out_prefix = self.default_l_out_prefix r_out_prefix = self.default_r_out_prefix # Check for l_out_prefix in args. if len(args) > 8: l_out_prefix = args[8] expected_output_attrs.append(l_out_prefix + args[2]) # Check for l_out_attrs in args. if len(args) > 6: if args[6]: for attr in args[6]: expected_output_attrs.append(l_out_prefix + attr) # Check for r_out_prefix in args. if len(args) > 9: r_out_prefix = args[9] expected_output_attrs.append(r_out_prefix + args[3]) # Check for r_out_attrs in args. if len(args) > 7: if args[7]: for attr in args[7]: expected_output_attrs.append(r_out_prefix + attr) # verify whether the output table has the necessary attributes. assert_list_equal(list(actual_candset.columns.values), expected_output_attrs) actual_pairs = set() for idx, row in actual_candset.iterrows(): actual_pairs.add(','.join((str(row[l_out_prefix + args[2]]), str(row[r_out_prefix + args[3]])))) # verify whether the actual pairs and the expected pairs match. assert_equal(len(expected_pairs), len(actual_pairs)) common_pairs = actual_pairs.intersection(expected_pairs) assert_equal(len(common_pairs), len(expected_pairs))
def test_invalid_r_out_attr(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr', 'B.attr', ['A.attr'], ['B.invalid_attr'])
def test_numeric_r_filter_attr(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr', 'B.int_attr')
def test_invalid_rtable(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, [], 'A.id', 'B.id', 'A.attr', 'B.attr')
def test_filter_tables(self, tokenizer, sim_measure_type, threshold, allow_empty, allow_missing, args): suffix_filter = SuffixFilter(tokenizer, sim_measure_type, threshold, allow_empty, allow_missing) sim_fn = get_sim_function(sim_measure_type) # compute the join output pairs join_output_pairs = set() for l_idx, l_row in args[0].iterrows(): for r_idx, r_row in args[1].iterrows(): # if allow_missing is set to True, then add pairs containing # missing value to the join output. if pd.isnull(l_row[args[4]]) or pd.isnull(r_row[args[5]]): if allow_missing: join_output_pairs.add(','.join((str(l_row[args[2]]), str(r_row[args[3]])))) continue if sim_measure_type == 'EDIT_DISTANCE': l_join_val = str(l_row[args[4]]) r_join_val = str(r_row[args[5]]) comp_fn = COMP_OP_MAP['<='] else: l_join_val = tokenizer.tokenize(str(l_row[args[4]])) r_join_val = tokenizer.tokenize(str(r_row[args[5]])) comp_fn = COMP_OP_MAP['>='] if (len(l_join_val) == 0 and len(r_join_val) == 0 and sim_measure_type not in ['OVERLAP', 'EDIT_DISTANCE']): if allow_empty: join_output_pairs.add(','.join((str(l_row[args[2]]), str(r_row[args[3]])))) continue # if both attributes are not missing and not empty, then check # if the pair satisfies the join condition. If yes, then add it # to the join output. if comp_fn(sim_fn(l_join_val, r_join_val), threshold): join_output_pairs.add(','.join((str(l_row[args[2]]), str(r_row[args[3]])))) actual_candset = suffix_filter.filter_tables(*args) expected_output_attrs = ['_id'] l_out_prefix = self.default_l_out_prefix r_out_prefix = self.default_r_out_prefix # Check for l_out_prefix in args. if len(args) > 8: l_out_prefix = args[8] expected_output_attrs.append(l_out_prefix + args[2]) # Check for r_out_prefix in args. if len(args) > 9: r_out_prefix = args[9] expected_output_attrs.append(r_out_prefix + args[3]) # Check for l_out_attrs in args. if len(args) > 6: if args[6]: l_out_attrs = remove_redundant_attrs(args[6], args[2]) for attr in l_out_attrs: expected_output_attrs.append(l_out_prefix + attr) # Check for r_out_attrs in args. if len(args) > 7: if args[7]: r_out_attrs = remove_redundant_attrs(args[7], args[3]) for attr in r_out_attrs: expected_output_attrs.append(r_out_prefix + attr) # verify whether the output table has the necessary attributes. assert_list_equal(list(actual_candset.columns.values), expected_output_attrs) actual_pairs = set() for idx, row in actual_candset.iterrows(): actual_pairs.add(','.join((str(int(row[l_out_prefix + args[2]])), str(int(row[r_out_prefix + args[3]]))))) # verify whether all the join output pairs are # present in the actual output pairs common_pairs = actual_pairs.intersection(join_output_pairs) assert_equal(len(common_pairs), len(join_output_pairs))
def test_invalid_r_out_attr(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr', 'B.attr', ['A.attr'], ['B.invalid_attr'])
def test_numeric_r_filter_attr(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, self.B, 'A.id', 'B.id', 'A.attr', 'B.int_attr')
def test_invalid_rtable(self): suffix_filter = SuffixFilter(self.tokenizer, self.sim_measure_type, self.threshold) suffix_filter.filter_tables(self.A, [], 'A.id', 'B.id', 'A.attr', 'B.attr')
def test_filter_tables(self, tokenizer, sim_measure_type, threshold, allow_empty, allow_missing, args): suffix_filter = SuffixFilter(tokenizer, sim_measure_type, threshold, allow_empty, allow_missing) sim_fn = get_sim_function(sim_measure_type) # compute the join output pairs join_output_pairs = set() for l_idx, l_row in args[0].iterrows(): for r_idx, r_row in args[1].iterrows(): # if allow_missing is set to True, then add pairs containing # missing value to the join output. if pd.isnull(l_row[args[4]]) or pd.isnull(r_row[args[5]]): if allow_missing: join_output_pairs.add(','.join( (str(l_row[args[2]]), str(r_row[args[3]])))) continue if sim_measure_type == 'EDIT_DISTANCE': l_join_val = str(l_row[args[4]]) r_join_val = str(r_row[args[5]]) comp_fn = COMP_OP_MAP['<='] else: l_join_val = tokenizer.tokenize(str(l_row[args[4]])) r_join_val = tokenizer.tokenize(str(r_row[args[5]])) comp_fn = COMP_OP_MAP['>='] if (len(l_join_val) == 0 and len(r_join_val) == 0 and sim_measure_type not in ['OVERLAP', 'EDIT_DISTANCE']): if allow_empty: join_output_pairs.add(','.join( (str(l_row[args[2]]), str(r_row[args[3]])))) continue # if both attributes are not missing and not empty, then check # if the pair satisfies the join condition. If yes, then add it # to the join output. if comp_fn(sim_fn(l_join_val, r_join_val), threshold): join_output_pairs.add(','.join( (str(l_row[args[2]]), str(r_row[args[3]])))) actual_candset = suffix_filter.filter_tables(*args) expected_output_attrs = ['_id'] l_out_prefix = self.default_l_out_prefix r_out_prefix = self.default_r_out_prefix # Check for l_out_prefix in args. if len(args) > 8: l_out_prefix = args[8] expected_output_attrs.append(l_out_prefix + args[2]) # Check for r_out_prefix in args. if len(args) > 9: r_out_prefix = args[9] expected_output_attrs.append(r_out_prefix + args[3]) # Check for l_out_attrs in args. if len(args) > 6: if args[6]: l_out_attrs = remove_redundant_attrs(args[6], args[2]) for attr in l_out_attrs: expected_output_attrs.append(l_out_prefix + attr) # Check for r_out_attrs in args. if len(args) > 7: if args[7]: r_out_attrs = remove_redundant_attrs(args[7], args[3]) for attr in r_out_attrs: expected_output_attrs.append(r_out_prefix + attr) # verify whether the output table has the necessary attributes. assert_list_equal(list(actual_candset.columns.values), expected_output_attrs) actual_pairs = set() for idx, row in actual_candset.iterrows(): actual_pairs.add(','.join((str(int(row[l_out_prefix + args[2]])), str(int(row[r_out_prefix + args[3]]))))) # verify whether all the join output pairs are # present in the actual output pairs common_pairs = actual_pairs.intersection(join_output_pairs) assert_equal(len(common_pairs), len(join_output_pairs))