def test_rb_block_tables(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') rb = mg.RuleBasedBlocker() feature_table = mg.get_features_for_blocking(A, B) rb.add_rule([ 'name_name_mel(ltuple, rtuple) < 0.4', 'birth_year_birth_year_lev(ltuple, rtuple) < 0.5' ], feature_table) rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) != 1'], feature_table) C = rb.block_tables(A, B, 'zipcode', 'zipcode') s1 = sorted( ['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode']) assert_equal(s1, sorted(C.columns)) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID') A['dummy'] = 1 B['dummy'] = 1 ab = mg.AttrEquivalenceBlocker() D = ab.block_tables(A, B, 'dummy', 'dummy') fv = mg.extract_feature_vecs(D, feature_table=feature_table) expected_ids = fv.ix[((fv.name_name_mel >= 0.4) | (fv.birth_year_birth_year_lev >= 0.5)) & fv.zipcode_zipcode_exm == 1, [ 'ltable.ID', 'rtable.ID', ]] actual_ids = C[['ltable.ID', 'rtable.ID']] ids_exp = list( expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) ids_act = list( actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) assert_equal(cmp(ids_exp, ids_act), 0)
def test_bb_block_tables(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') bb = mg.BlackBoxBlocker() bb.set_black_box_function(block_fn) C = bb.block_tables(A, B, 'zipcode', 'zipcode') s1 = sorted( ['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode']) assert_equal(s1, sorted(C.columns)) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID') feature_table = mg.get_features_for_blocking(A, B) A['dummy'] = 1 B['dummy'] = 1 ab = mg.AttrEquivalenceBlocker() D = ab.block_tables(A, B, 'dummy', 'dummy') fv = mg.extract_feature_vecs(D, feature_table=feature_table) expected_ids = fv.ix[(fv.name_name_mel >= 0.4), ['ltable.ID', 'rtable.ID']] actual_ids = C[['ltable.ID', 'rtable.ID']] ids_exp = list( expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) ids_act = list( actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) assert_equal(cmp(ids_exp, ids_act), 0)
def test_ab_block_tuples(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() assert_equal(ab.block_tuples(A.ix[1], B.ix[2], 'zipcode', 'zipcode'), False) assert_equal(ab.block_tuples(A.ix[2], B.ix[2], 'zipcode', 'zipcode'), True)
def test_ab_block_tables_wi_no_tuples(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() C = ab.block_tables(A, B, 'name', 'name') assert_equal(len(C), 0) assert_equal(sorted(C.columns), sorted(['_id', 'ltable.ID', 'rtable.ID'])) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')
def test_save_load_ab_blocker(): filename = '__mg_obj__.pkl' ab0 = mg.AttrEquivalenceBlocker() mg.save_object(ab0, filename) ab1 = mg.load_object(filename) try: os.remove(filename) except OSError: pass assert_equal(type(ab0), type(ab1))
def test_bl_combiner_wi_no_tuples(): A = mg.read_csv(path_for_A, key='ID') B = mg.read_csv(path_for_B, key='ID') ab = mg.AttrEquivalenceBlocker() C1 = ab.block_tables(A, B, 'name', 'name') C = mg.combine_block_outputs_via_union([C1, C1, C1]) s = ['_id', 'ltable.ID', 'rtable.ID'] assert_equal(sorted(s) == sorted(C.columns), True) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')
def test_bb_block_candset_wi_no_tuples(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() C = ab.block_tables(A, B, 'birth_year', 'birth_year') bb = mg.BlackBoxBlocker() bb.set_black_box_function(evil_block_fn) D = bb.block_candset(C) assert_equal(len(D), 0) assert_equal(sorted(D.columns), sorted(['_id', 'ltable.ID', 'rtable.ID'])) assert_equal(D.get_key(), '_id') assert_equal(D.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(D.get_property('foreign_key_rtable'), 'rtable.ID')
def test_bl_combiner_wi_no_fields(): A = mg.read_csv(path_for_A, key='ID') B = mg.read_csv(path_for_B, key='ID') ab = mg.AttrEquivalenceBlocker() C1 = ab.block_tables(A, B, 'zipcode', 'zipcode') C2 = ab.block_tables(A, B, 'hourly_wage', 'hourly_wage') C3 = ab.block_tables(A, B, 'birth_year', 'birth_year') C = mg.combine_block_outputs_via_union([C1, C2, C3]) s = ['_id', 'ltable.ID', 'rtable.ID'] assert_equal(sorted(s) == sorted(C.columns), True) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')
def test_rb_block_candset_wi_no_tuples(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() C = ab.block_tables(A, B, 'birth_year', 'birth_year') rb = mg.RuleBasedBlocker() feature_table = mg.get_features_for_blocking(A, B) rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) >= 0'], feature_table) D = rb.block_candset(C) assert_equal(len(D), 0) assert_equal(sorted(D.columns), sorted(['_id', 'ltable.ID', 'rtable.ID'])) assert_equal(D.get_key(), '_id') assert_equal(D.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(D.get_property('foreign_key_rtable'), 'rtable.ID')
def test_ab_block_tables(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() C = ab.block_tables(A, B, 'zipcode', 'zipcode', 'zipcode', 'zipcode') s1 = sorted( ['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode']) assert_equal(s1, sorted(C.columns)) assert_equal(C.get_key(), '_id') assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID') assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID') k1 = np.array(C[['ltable.zipcode']]) k2 = np.array(C[['rtable.zipcode']]) assert_equal(all(k1 == k2), True)
def test_tocsv_vtable_wi_props(): filename = '__tbl__.csv' A = mg.read_csv(path_for_A, key='ID') B = mg.read_csv(path_for_B, key='ID') ab = mg.AttrEquivalenceBlocker() C1 = ab.block_tables(A, B, 'zipcode', 'zipcode') C1.to_csv(filename, suppress_properties=False) C2 = mg.read_csv(filename, ltable=A, rtable=B) try: os.remove(filename) except OSError: pass assert_equal(C2.get_key(), C1.get_key()) assert_equal(C2.get_property('foreign_key_ltable'), C1.get_property('foreign_key_ltable')) assert_equal(C2.get_property('foreign_key_rtable'), C1.get_property('foreign_key_rtable'))
def test_bb_block_candset(): A = mg.load_dataset('table_A') B = mg.load_dataset('table_B') ab = mg.AttrEquivalenceBlocker() E = ab.block_tables(A, B, 'zipcode', 'zipcode') bb = mg.BlackBoxBlocker() bb.set_black_box_function(block_fn) C = bb.block_candset(E) feature_table = mg.get_features_for_blocking(A, B) fv = mg.extract_feature_vecs(C, feature_table=feature_table) expected_ids = fv.ix[(fv.name_name_mel >= 0.4), ['ltable.ID', 'rtable.ID']] actual_ids = C[['ltable.ID', 'rtable.ID']] ids_exp = list( expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) ids_act = list( actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values) assert_equal(cmp(ids_exp, ids_act), 0)