def test_ml_matcher_inplace_false_predict(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
train.drop('ltable.id', axis=1, inplace=True)
train.drop('rtable.id', axis=1, inplace=True)
test.drop('ltable.id', axis=1, inplace=True)
test.drop('rtable.id', axis=1, inplace=True)
test.drop('gold', axis=1, inplace=True)
dt.fit(table=train, exclude_attrs='_id', target_attr='gold')
predictions = dt.predict(table=test,
exclude_attrs='_id',
target_attr='predicted',
inplace=False,
append=True)
self.assertNotEqual(id(predictions), id(test))
self.assertEqual(len(predictions), len(test))
self.assertEqual(
set(list(test.columns)).issubset(list(predictions.columns)), True)
p_col = predictions.columns[len(predictions.columns) - 1]
self.assertEqual(p_col, 'predicted')
def test_ml_matcher_invalid_input_combn_fit(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(x=train, table=train)
def test_ml_matcher_invalid_input_combn_fit(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(x=train, table=train)
def test_ml_matcher_invalid_df_predict(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train, exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'], target_attr='gold')
predictions = dt.predict(table="", exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
def test_ml_matcher_target_attr_not_present_fit(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
train.drop('ltable.id', axis=1, inplace=True)
train.drop('rtable.id', axis=1, inplace=True)
test.drop('ltable.id', axis=1, inplace=True)
test.drop('rtable.id', axis=1, inplace=True)
dt.fit(table=train, exclude_attrs='_id', target_attr='gold1')
def test_ml_matcher_target_attr_not_present_fit(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
train.drop('ltable.id', axis=1, inplace=True)
train.drop('rtable.id', axis=1, inplace=True)
test.drop('ltable.id', axis=1, inplace=True)
test.drop('rtable.id', axis=1, inplace=True)
dt.fit(table=train, exclude_attrs='_id', target_attr='gold1')
def test_ml_matcher_invalid_df_predict(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train,
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='gold')
predictions = dt.predict(
table="",
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
def test_visualize_tree_invalid_df(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C, feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors, exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
visualize_tree(dt.clf, feature_vectors.columns, exclude_attrs=['_id', 'ltable_ID',
'rtable_ID', 'labels'])
def test_ml_matcher_valid_1(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train, exclude_attrs=['ltable.id', 'rtable.id', '_id'], target_attr='gold')
predictions = dt.predict(table=test, exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
self.assertEqual(len(predictions), len(test))
self.assertEqual(set(list(predictions.columns)).issubset(list(test.columns)), True)
p_col = predictions.columns[len(predictions.columns)-1]
self.assertEqual(p_col, 'predicted')
def test_debug_dt_matcher_valid(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C, feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors, exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
debug_decisiontree_matcher(dt, A.ix[1], B.ix[2], feat_table=feature_table,
fv_columns=feature_vectors.columns,
exclude_attrs=['ltable_ID', 'rtable_ID', '_id', 'labels'])
def test_ml_matcher_append_false_predict(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
train.drop('ltable.id', axis=1, inplace=True)
train.drop('rtable.id', axis=1, inplace=True)
test.drop('ltable.id', axis=1, inplace=True)
test.drop('rtable.id', axis=1, inplace=True)
test.drop('gold', axis=1, inplace=True)
dt.fit(table=train, exclude_attrs='_id', target_attr='gold')
predictions = dt.predict(table=test, exclude_attrs='_id',
target_attr='predicted',
append=False)
self.assertEqual(len(predictions), len(test))
def test_ml_matcher_valid_2(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
col_list = list(feature_vectors.columns)
l = list_diff(col_list, [cm.get_key(feature_vectors), cm.get_fk_ltable(feature_vectors),
cm.get_fk_rtable(feature_vectors),
'gold'])
X = train[l]
Y = train['gold']
dt.fit(x=X, y=Y)
predictions = dt.predict(test[l])
self.assertEqual(len(predictions), len(test))
def test_vis_tuple_debug_dt_matcher_valid_2(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C, feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors, exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
s = pd.DataFrame(feature_vectors.ix[0])
s1 = s.T
vis_tuple_debug_dt_matcher(dt.clf, s1,
exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'])
def test_ml_matcher_valid_with_id_in_y(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
col_list = list(feature_vectors.columns)
l = list_diff(col_list, [
cm.get_fk_ltable(feature_vectors),
cm.get_fk_rtable(feature_vectors), 'gold'
])
X = train[l]
Y = train[['_id', 'gold']]
dt.fit(x=X, y=Y)
predictions = dt.predict(test[l])
self.assertEqual(len(predictions), len(test))
def test_visualize_tree_invalid_df(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C,
feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors,
exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
visualize_tree(
dt.clf,
feature_vectors.columns,
exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'])
def test_ml_matcher_target_attr_present_in_ex_attrs(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train,
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='gold')
predictions = dt.predict(
table=test,
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
self.assertEqual(len(predictions), len(test))
l = len(set(list(predictions.columns)).difference(list(test.columns)))
self.assertEqual(l, 0)
p_col = predictions.columns[len(predictions.columns) - 1]
self.assertEqual(p_col, 'predicted')
def test_ml_matcher_ex_attrs_not_list(self):
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
train.drop('ltable.id', axis=1, inplace=True)
train.drop('rtable.id', axis=1, inplace=True)
test.drop('ltable.id', axis=1, inplace=True)
test.drop('rtable.id', axis=1, inplace=True)
dt.fit(table=train, exclude_attrs='_id', target_attr='gold')
predictions = dt.predict(table=test, exclude_attrs=['_id', 'gold'],
target_attr='predicted',
append=True)
self.assertEqual(len(predictions), len(test))
l = len(set(list(predictions.columns)).difference(list(test.columns)))
self.assertEqual(l, 0)
p_col = predictions.columns[len(predictions.columns)-1]
self.assertEqual(p_col, 'predicted')
def test_vis_tuple_debug_dt_matcher_valid_1(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C,
feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors,
exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
s = pd.DataFrame(feature_vectors.ix[0])
s1 = s.T
vis_tuple_debug_dt_matcher(
dt, s1, exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'])
def test_debug_dt_matcher_valid(self):
A = read_csv_metadata(path_a)
B = read_csv_metadata(path_b, key='ID')
C = read_csv_metadata(path_c, ltable=A, rtable=B)
labels = [0] * 7
labels.extend([1] * 8)
C['labels'] = labels
feature_table = get_features_for_matching(A, B)
feature_vectors = extract_feature_vecs(C,
feature_table=feature_table,
attrs_after='labels')
dt = DTMatcher()
dt.fit(table=feature_vectors,
exclude_attrs=['_id', 'ltable_ID', 'rtable_ID', 'labels'],
target_attr='labels')
debug_decisiontree_matcher(
dt,
A.ix[1],
B.ix[2],
feat_table=feature_table,
fv_columns=feature_vectors.columns,
exclude_attrs=['ltable_ID', 'rtable_ID', '_id', 'labels'])
def test_ml_matcher_invalid_df(self):
dt = DTMatcher(name='DecisionTree')
dt.fit(table="",
exclude_attrs=['ltable.id', 'rtable.id', '_id'],
target_attr='gold')
def test_ml_matcher_invalid_df_1(self):
dt = DTMatcher(name='DecisionTree')
dt.fit(x="", y="")
import os
import magellan.matcher.matcherutils as mu
from magellan.io.parsers import read_csv_metadata
from magellan.matcher.dtmatcher import DTMatcher
from magellan.utils.generic_helper import get_install_path
feat_datasets_path = os.sep.join([get_install_path(), 'datasets', 'test_datasets', 'matcherselector'])
fpath_a = os.sep.join([feat_datasets_path, 'DBLP_demo.csv'])
fpath_b = os.sep.join([feat_datasets_path, 'ACM_demo.csv'])
fpath_c = os.sep.join([feat_datasets_path, 'dblp_acm_demo_labels.csv'])
fpath_f = os.sep.join([feat_datasets_path, 'feat_vecs.csv'])
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train, exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'], target_attr='gold')
predictions = dt.predict(table=test, exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
print('Done')
def test_ml_matcher_invalid_df_1(self):
dt = DTMatcher(name='DecisionTree')
dt.fit(x="", y="")
import os
import magellan.matcher.matcherutils as mu
from magellan.io.parsers import read_csv_metadata
from magellan.matcher.dtmatcher import DTMatcher
from magellan.utils.generic_helper import get_install_path
feat_datasets_path = os.sep.join(
[get_install_path(), 'datasets', 'test_datasets', 'matcherselector'])
fpath_a = os.sep.join([feat_datasets_path, 'DBLP_demo.csv'])
fpath_b = os.sep.join([feat_datasets_path, 'ACM_demo.csv'])
fpath_c = os.sep.join([feat_datasets_path, 'dblp_acm_demo_labels.csv'])
fpath_f = os.sep.join([feat_datasets_path, 'feat_vecs.csv'])
A = read_csv_metadata(fpath_a, key='id')
B = read_csv_metadata(fpath_b, key='id')
feature_vectors = read_csv_metadata(fpath_f, ltable=A, rtable=B)
train_test = mu.train_test_split(feature_vectors)
train, test = train_test['train'], train_test['test']
dt = DTMatcher(name='DecisionTree')
dt.fit(table=train,
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='gold')
predictions = dt.predict(
table=test,
exclude_attrs=['ltable.id', 'rtable.id', '_id', 'gold'],
target_attr='predicted',
append=True)
print('Done')
def test_ml_matcher_invalid_df(self):
dt = DTMatcher(name='DecisionTree')
dt.fit(table="", exclude_attrs=['ltable.id', 'rtable.id', '_id'], target_attr='gold')