Python get_features_for_blocking Exemples

Exemple #1

Fichier : save_load_tests.py Projet : Yashg19/enrique

def test_save_load_rb_blocker():
    #mg.init_jvm()
    filename = '__mg_obj__.pkl'
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    feature_table = mg.get_features_for_blocking(A, B)
    rb0 = mg.RuleBasedBlocker()
    rb0.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) == 1'], feature_table)
    rb0.add_rule(['birth_year_birth_year_anm(ltuple, rtuple) > 0.95',
                  'name_name_mel(ltuple, rtuple)> 0.4'],
                 feature_table)
    C0 = rb0.block_tables(A, B)
    mg.save_object(rb0, filename)
    rb1 = mg.load_object(filename)
    try:
        os.remove(filename)
    except OSError:
        pass

    assert_equal(type(rb0), type(rb1))
    assert_equal(len(rb0.rules), len(rb1.rules))
    assert_equal(len(rb0.rule_source), len(rb1.rule_source))
    assert_equal(rb0.rule_cnt, rb1.rule_cnt)

    C1 = rb1.block_tables(A, B)
    assert_equal(len(C0), len(C1))
    assert_equal(sorted(C0.columns), sorted(C0.columns))

Exemple #2

Fichier : rule_based_blocker_tests.py Projet : epaulson/enrique

def test_rb_block_tables():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule([
        'name_name_mel(ltuple, rtuple) < 0.4',
        'birth_year_birth_year_lev(ltuple, rtuple) < 0.5'
    ], feature_table)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) != 1'], feature_table)
    C = rb.block_tables(A, B, 'zipcode', 'zipcode')
    s1 = sorted(
        ['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode'])
    assert_equal(s1, sorted(C.columns))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

    A['dummy'] = 1
    B['dummy'] = 1
    ab = mg.AttrEquivalenceBlocker()
    D = ab.block_tables(A, B, 'dummy', 'dummy')
    fv = mg.extract_feature_vecs(D, feature_table=feature_table)
    expected_ids = fv.ix[((fv.name_name_mel >= 0.4) |
                          (fv.birth_year_birth_year_lev >= 0.5))
                         & fv.zipcode_zipcode_exm == 1, [
                             'ltable.ID',
                             'rtable.ID',
                         ]]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(
        expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(
        actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)

Exemple #3

Fichier : save_load_tests.py Projet : Yashg19/enrique

def test_save_load_feature_table():
    #mg.init_jvm()

    filename = '__mg_obj__.pkl'
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    feature_table0 = mg.get_features_for_blocking(A, B)
    mg.save_object(feature_table0, filename)
    feature_table1 = mg.load_object(filename)
    try:
        os.remove(filename)
    except OSError:
        pass
    assert_equal(type(feature_table0), type(feature_table1))
    assert_equal(len(feature_table0), len(feature_table1))
    assert sorted(feature_table0.columns) == sorted(feature_table1.columns)
    ft0_functions = list(feature_table0['function'])
    ft1_functions = list(feature_table1['function'])
    for f0, f1 in zip(ft0_functions, ft1_functions):
        a = f0(A.ix[1], B.ix[2])
        b = f1(A.ix[1], B.ix[2])
        if math.isnan(a) == False and math.isnan(b) == False:
            assert_equal(a, b)
        if math.isnan(a) == True:
            assert_equal(math.isnan(b), True)

Exemple #4

Fichier : save_load_tests.py Projet : epaulson/enrique

def test_save_load_rb_blocker():
    #mg.init_jvm()
    filename = '__mg_obj__.pkl'
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    feature_table = mg.get_features_for_blocking(A, B)
    rb0 = mg.RuleBasedBlocker()
    rb0.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) == 1'], feature_table)
    rb0.add_rule([
        'birth_year_birth_year_anm(ltuple, rtuple) > 0.95',
        'name_name_mel(ltuple, rtuple)> 0.4'
    ], feature_table)
    C0 = rb0.block_tables(A, B)
    mg.save_object(rb0, filename)
    rb1 = mg.load_object(filename)
    try:
        os.remove(filename)
    except OSError:
        pass

    assert_equal(type(rb0), type(rb1))
    assert_equal(len(rb0.rules), len(rb1.rules))
    assert_equal(len(rb0.rule_source), len(rb1.rule_source))
    assert_equal(rb0.rule_cnt, rb1.rule_cnt)

    C1 = rb1.block_tables(A, B)
    assert_equal(len(C0), len(C1))
    assert_equal(sorted(C0.columns), sorted(C0.columns))

Exemple #5

Fichier : black_box_blocker_tests.py Projet : epaulson/enrique

def test_bb_block_tables():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    bb = mg.BlackBoxBlocker()
    bb.set_black_box_function(block_fn)
    C = bb.block_tables(A, B, 'zipcode', 'zipcode')
    s1 = sorted(
        ['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode'])
    assert_equal(s1, sorted(C.columns))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

    feature_table = mg.get_features_for_blocking(A, B)
    A['dummy'] = 1
    B['dummy'] = 1
    ab = mg.AttrEquivalenceBlocker()
    D = ab.block_tables(A, B, 'dummy', 'dummy')
    fv = mg.extract_feature_vecs(D, feature_table=feature_table)
    expected_ids = fv.ix[(fv.name_name_mel >= 0.4), ['ltable.ID', 'rtable.ID']]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(
        expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(
        actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)

Exemple #6

Fichier : save_load_tests.py Projet : epaulson/enrique

def test_save_load_feature_table():
    #mg.init_jvm()

    filename = '__mg_obj__.pkl'
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    feature_table0 = mg.get_features_for_blocking(A, B)
    mg.save_object(feature_table0, filename)
    feature_table1 = mg.load_object(filename)
    try:
        os.remove(filename)
    except OSError:
        pass
    assert_equal(type(feature_table0), type(feature_table1))
    assert_equal(len(feature_table0), len(feature_table1))
    assert sorted(feature_table0.columns) == sorted(feature_table1.columns)
    ft0_functions = list(feature_table0['function'])
    ft1_functions = list(feature_table1['function'])
    for f0, f1 in zip(ft0_functions, ft1_functions):
        a = f0(A.ix[1], B.ix[2])
        b = f1(A.ix[1], B.ix[2])
        if math.isnan(a) == False and math.isnan(b) == False:
            assert_equal(a, b)
        if math.isnan(a) == True:
            assert_equal(math.isnan(b), True)

Exemple #7

Fichier : rule_based_blocker_tests.py Projet : Yashg19/enrique

def test_rb_block_tables():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['name_name_mel(ltuple, rtuple) < 0.4',
                 'birth_year_birth_year_lev(ltuple, rtuple) < 0.5'],
                feature_table)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) != 1'],
                feature_table)
    C = rb.block_tables(A, B, 'zipcode', 'zipcode')
    s1 = sorted(['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode'])
    assert_equal(s1, sorted(C.columns))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

    A['dummy'] = 1
    B['dummy'] = 1
    ab = mg.AttrEquivalenceBlocker()
    D = ab.block_tables(A, B, 'dummy','dummy')
    fv = mg.extract_feature_vecs(D,  feature_table=feature_table)
    expected_ids = fv.ix[((fv.name_name_mel >= 0.4) | (fv.birth_year_birth_year_lev >= 0.5)) &
      fv.zipcode_zipcode_exm == 1 ,
      ['ltable.ID', 'rtable.ID',
       ]]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)

Exemple #8

Fichier : rule_based_blocker_tests.py Projet : epaulson/enrique

def test_rb_block_tables_wi_no_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) >= 0'], feature_table)
    C = rb.block_tables(A, B)
    assert_equal(len(C), 0)
    assert_equal(sorted(C.columns), sorted(['_id', 'ltable.ID', 'rtable.ID']))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

Exemple #9

Fichier : rule_based_blocker_tests.py Projet : epaulson/enrique

def test_rb_block_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule([
        'name_name_mel(ltuple, rtuple) < 0.4',
        'birth_year_birth_year_lev(ltuple, rtuple) < 0.5'
    ], feature_table)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) != 1'], feature_table)

    assert_equal(rb.block_tuples(A.ix[0], B.ix[0]), False)
    assert_equal(rb.block_tuples(A.ix[1], B.ix[1]), True)

Exemple #10

Fichier : rule_based_blocker_tests.py Projet : Yashg19/enrique

def test_rb_block_tables_wi_no_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) >= 0'],
                feature_table)
    C = rb.block_tables(A, B)
    assert_equal(len(C),  0)
    assert_equal(sorted(C.columns), sorted(['_id', 'ltable.ID', 'rtable.ID']))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

Exemple #11

Fichier : rule_based_blocker_tests.py Projet : Yashg19/enrique

def test_rb_block_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['name_name_mel(ltuple, rtuple) < 0.4',
                 'birth_year_birth_year_lev(ltuple, rtuple) < 0.5'],
                feature_table)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) != 1'],
                feature_table)

    assert_equal(rb.block_tuples(A.ix[0], B.ix[0]), False)
    assert_equal(rb.block_tuples(A.ix[1], B.ix[1]), True)

Exemple #12

Fichier : rule_based_blocker_tests.py Projet : epaulson/enrique

def test_rb_block_candset_wi_no_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    ab = mg.AttrEquivalenceBlocker()
    C = ab.block_tables(A, B, 'birth_year', 'birth_year')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) >= 0'], feature_table)
    D = rb.block_candset(C)
    assert_equal(len(D), 0)
    assert_equal(sorted(D.columns), sorted(['_id', 'ltable.ID', 'rtable.ID']))
    assert_equal(D.get_key(), '_id')
    assert_equal(D.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(D.get_property('foreign_key_rtable'), 'rtable.ID')

Exemple #13

Fichier : rule_based_blocker_tests.py Projet : Yashg19/enrique

def test_rb_block_candset_wi_no_tuples():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    ab = mg.AttrEquivalenceBlocker()
    C = ab.block_tables(A, B, 'birth_year', 'birth_year')
    rb = mg.RuleBasedBlocker()
    feature_table = mg.get_features_for_blocking(A, B)
    rb.add_rule(['zipcode_zipcode_exm(ltuple, rtuple) >= 0'],
                feature_table)
    D = rb.block_candset(C)
    assert_equal(len(D),  0)
    assert_equal(sorted(D.columns), sorted(['_id', 'ltable.ID', 'rtable.ID']))
    assert_equal(D.get_key(), '_id')
    assert_equal(D.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(D.get_property('foreign_key_rtable'), 'rtable.ID')

Exemple #14

Fichier : black_box_blocker_tests.py Projet : Yashg19/enrique

def test_bb_block_candset():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    ab = mg.AttrEquivalenceBlocker()
    E = ab.block_tables(A, B, 'zipcode', 'zipcode')
    bb = mg.BlackBoxBlocker()
    bb.set_black_box_function(block_fn)
    C = bb.block_candset(E)
    feature_table = mg.get_features_for_blocking(A, B)
    fv = mg.extract_feature_vecs(C, feature_table=feature_table)
    expected_ids = fv.ix[(fv.name_name_mel >= 0.4) ,
      ['ltable.ID', 'rtable.ID']]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)

Exemple #15

Fichier : black_box_blocker_tests.py Projet : epaulson/enrique

def test_bb_block_candset():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    ab = mg.AttrEquivalenceBlocker()
    E = ab.block_tables(A, B, 'zipcode', 'zipcode')
    bb = mg.BlackBoxBlocker()
    bb.set_black_box_function(block_fn)
    C = bb.block_candset(E)
    feature_table = mg.get_features_for_blocking(A, B)
    fv = mg.extract_feature_vecs(C, feature_table=feature_table)
    expected_ids = fv.ix[(fv.name_name_mel >= 0.4), ['ltable.ID', 'rtable.ID']]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(
        expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(
        actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)

Exemple #16

Fichier : black_box_blocker_tests.py Projet : Yashg19/enrique

def test_bb_block_tables():
    A = mg.load_dataset('table_A')
    B = mg.load_dataset('table_B')
    bb = mg.BlackBoxBlocker()
    bb.set_black_box_function(block_fn)
    C = bb.block_tables(A, B, 'zipcode', 'zipcode')
    s1 = sorted(['_id', 'ltable.ID', 'rtable.ID', 'ltable.zipcode', 'rtable.zipcode'])
    assert_equal(s1, sorted(C.columns))
    assert_equal(C.get_key(), '_id')
    assert_equal(C.get_property('foreign_key_ltable'), 'ltable.ID')
    assert_equal(C.get_property('foreign_key_rtable'), 'rtable.ID')

    feature_table = mg.get_features_for_blocking(A, B)
    A['dummy'] = 1
    B['dummy'] = 1
    ab = mg.AttrEquivalenceBlocker()
    D = ab.block_tables(A, B, 'dummy','dummy')
    fv = mg.extract_feature_vecs(D,  feature_table=feature_table)
    expected_ids = fv.ix[(fv.name_name_mel >= 0.4) ,
      ['ltable.ID', 'rtable.ID']]
    actual_ids = C[['ltable.ID', 'rtable.ID']]
    ids_exp = list(expected_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    ids_act = list(actual_ids.set_index(['ltable.ID', 'rtable.ID']).index.values)
    assert_equal(cmp(ids_exp, ids_act), 0)