def test_parse_feat_str_parse_valid_2(self):
feature_string = "jaccard(qgm_3(ltuple['zipcode']), qgm_3(ltuple['zipcode']))"
p_dict = parse_feat_str(feature_string, get_tokenizers_for_matching(),
get_sim_funs_for_matching())
self.assertEqual(p_dict['left_attr_tokenizer'], 'qgm_3')
self.assertEqual(p_dict['right_attr_tokenizer'], 'qgm_3')
self.assertEqual(p_dict['simfunction'], 'jaccard')
def test_add_feature_invalid_df_columns(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(), get_sim_funs_for_matching())
add_feature(pd.DataFrame(), 'test', f_dict)
def test_parse_feat_str_parse_valid_1(self):
feature_string = "jaccard(qgm_3(ltuple['zipcode']), qgm_3(rtuple['zipcode']))"
p_dict = parse_feat_str(feature_string, get_tokenizers_for_matching(), get_sim_funs_for_matching())
self.assertEqual(p_dict['left_attr_tokenizer'], 'qgm_3')
self.assertEqual(p_dict['right_attr_tokenizer'], 'qgm_3')
self.assertEqual(p_dict['simfunction'], 'jaccard')
self.assertEqual(p_dict['left_attribute'], 'zipcode')
self.assertEqual(p_dict['right_attribute'], 'zipcode')
def test_add_feature_invalid_df_columns(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(),
get_sim_funs_for_matching())
add_feature(pd.DataFrame(), 'test', f_dict)
def test_get_features_invalid_ltable_rtable_switch(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
l_attr_types = afg.get_attr_types(A)
r_attr_types = afg.get_attr_types(B)
attr_corres = afg.get_attr_corres(B, A)
tok = get_tokenizers_for_matching()
sim = get_sim_funs_for_matching()
feat_table = afg.get_features(A, B, l_attr_types, r_attr_types, attr_corres, tok, sim)
def test_add_feature_name_already_present(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_table = create_feature_table()
len1 = len(feature_table)
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(), get_sim_funs_for_matching())
add_feature(feature_table, 'test', f_dict)
add_feature(feature_table, 'test', f_dict)
def test_add_feature_name_already_present(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_table = create_feature_table()
len1 = len(feature_table)
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(),
get_sim_funs_for_matching())
add_feature(feature_table, 'test', f_dict)
add_feature(feature_table, 'test', f_dict)
def test_get_features_invalid_ltable_rtable_switch(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
l_attr_types = afg.get_attr_types(A)
r_attr_types = afg.get_attr_types(B)
attr_corres = afg.get_attr_corres(B, A)
tok = get_tokenizers_for_matching()
sim = get_sim_funs_for_matching()
feat_table = afg.get_features(A, B, l_attr_types, r_attr_types,
attr_corres, tok, sim)
def test_add_features_valid_1(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_table = get_features_for_matching(A, B)
len1 = len(feature_table)
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(), get_sim_funs_for_matching())
add_feature(feature_table, 'test', f_dict)
len2 = len(feature_table)
self.assertEqual(len1+1, len2)
self.assertEqual(feature_table.ix[len(feature_table)-1, 'function'](A.ix[1], B.ix[2]), 1.0)
def test_add_features_valid_1(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
feature_table = get_features_for_matching(A, B)
len1 = len(feature_table)
feature_string = "exact_match(ltuple['zipcode'], rtuple['zipcode'])"
f_dict = get_feature_fn(feature_string, get_tokenizers_for_matching(),
get_sim_funs_for_matching())
add_feature(feature_table, 'test', f_dict)
len2 = len(feature_table)
self.assertEqual(len1 + 1, len2)
self.assertEqual(
feature_table.ix[len(feature_table) - 1, 'function'](A.ix[1],
B.ix[2]), 1.0)
def test_get_features_valid(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
l_attr_types = afg.get_attr_types(A)
r_attr_types = afg.get_attr_types(B)
attr_corres = afg.get_attr_corres(A, B)
tok = get_tokenizers_for_matching()
sim = get_sim_funs_for_matching()
feat_table = afg.get_features(A, B, l_attr_types, r_attr_types, attr_corres, tok, sim)
self.assertEqual(isinstance(feat_table, pd.DataFrame), True)
functions = feat_table['function']
for f in functions:
x = f(A.ix[1], B.ix[2])
self.assertEqual(x >= 0, True)
def test_get_features_valid(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
l_attr_types = afg.get_attr_types(A)
r_attr_types = afg.get_attr_types(B)
attr_corres = afg.get_attr_corres(A, B)
tok = get_tokenizers_for_matching()
sim = get_sim_funs_for_matching()
feat_table = afg.get_features(A, B, l_attr_types, r_attr_types,
attr_corres, tok, sim)
self.assertEqual(isinstance(feat_table, pd.DataFrame), True)
functions = feat_table['function']
for f in functions:
x = f(A.ix[1], B.ix[2])
self.assertEqual(x >= 0, True)
def get_features_for_matching(A, B):
if not isinstance(A, pd.DataFrame):
logger.error('Input table A is not of type pandas dataframe')
raise AssertionError('Input table A is not of type pandas dataframe')
if not isinstance(B, pd.DataFrame):
logger.error('Input table B is not of type pandas dataframe')
raise AssertionError('Input table B is not of type pandas dataframe')
sim_funcs = get_sim_funs_for_matching()
tok_funcs = get_tokenizers_for_matching()
t_A = get_attr_types(A)
t_B = get_attr_types(B)
attr_corres = get_attr_corres(A, B)
feat_table = get_features(A, B, t_A, t_B, attr_corres, tok_funcs, sim_funcs)
# export important variables to global name space
mg._match_t = tok_funcs
mg._match_s = sim_funcs
mg._atypes1 = t_A
mg._atypes2 = t_B
mg._match_c = attr_corres
return feat_table
def get_features_for_matching(A, B):
if not isinstance(A, pd.DataFrame):
logger.error('Input table A is not of type pandas dataframe')
raise AssertionError('Input table A is not of type pandas dataframe')
if not isinstance(B, pd.DataFrame):
logger.error('Input table B is not of type pandas dataframe')
raise AssertionError('Input table B is not of type pandas dataframe')
sim_funcs = get_sim_funs_for_matching()
tok_funcs = get_tokenizers_for_matching()
t_A = get_attr_types(A)
t_B = get_attr_types(B)
attr_corres = get_attr_corres(A, B)
feat_table = get_features(A, B, t_A, t_B, attr_corres, tok_funcs,
sim_funcs)
# export important variables to global name space
mg._match_t = tok_funcs
mg._match_s = sim_funcs
mg._atypes1 = t_A
mg._atypes2 = t_B
mg._match_c = attr_corres
return feat_table
def test_get_tokenizers_for_matching(self):
x = tok.get_tokenizers_for_matching()
self.assertEqual(isinstance(x, dict), True)
input = 'data science'
for name, value in six.iteritems(x):
self.assertEqual(isinstance(value(input), list), True)
def test_parse_feat_str_parse_exp(self):
feature_string = "jaccard~(qgm_3(ltuple[['zipcode']), qgm_3(rtuple['zipcode']))"
p_dict = parse_feat_str(feature_string, get_tokenizers_for_matching(), get_sim_funs_for_matching())
for k,v in six.iteritems(p_dict):
self.assertEqual(v, 'PARSE_EXP')
def test_parse_feat_str_parse_exp(self):
feature_string = "jaccard~(qgm_3(ltuple[['zipcode']), qgm_3(rtuple['zipcode']))"
p_dict = parse_feat_str(feature_string, get_tokenizers_for_matching(),
get_sim_funs_for_matching())
for k, v in six.iteritems(p_dict):
self.assertEqual(v, 'PARSE_EXP')
def test_get_tokenizers_for_matching_invalid(self):
x = tok.get_tokenizers_for_matching(None, None)