def extract_features(ltable_df, rtable_df, candset_df): tokenizers = em.get_tokenizers_for_matching() sim_functions = em.get_sim_funs_for_matching() left_attr_types = em.get_attr_types(ltable_df) right_attr_types = em.get_attr_types(rtable_df) correspondences = em.get_attr_corres(ltable_df, rtable_df) feature_dict_list = [] attribute_type_rank = {'boolean':1, 'numeric':2, 'str_eq_1w':3, 'str_bt_1w_5w':4, 'str_bt_5w_10w':5, 'str_gt_10w':6, 'un_determined':7} for c in correspondences['corres']: if left_attr_types[c[0]] != right_attr_types[c[1]]: if attribute_type_rank[left_attr_types[c[0]]] < attribute_type_rank[right_attr_types[c[1]]]: left_attr_types[c[0]] = right_attr_types[c[1]] else: right_attr_types[c[1]] = left_attr_types[c[0]] feature_records = get_features(ltable_df,rtable_df,left_attr_types, right_attr_types, correspondences, tokenizers, sim_functions) #Remove all features based on id - they are often useless feature_records = feature_records[feature_records.left_attribute !='id'] feature_records.reset_index(inplace=True,drop=True) distance_functions = ["lev_dist", "rdf"] non_normalized_functions = ["aff", "sw", "swn", "nmw"] keep_features = [True]*feature_records.shape[0] for i in range(feature_records.shape[0]): feature = feature_records.loc[i,"feature_name"] for func in distance_functions + non_normalized_functions: if func in feature: keep_features[i] = False feature_records = feature_records.loc[keep_features,:] print("\n\nExtracting the full set of features:") candset_features_df = em.extract_feature_vecs(candset_df,feature_table=feature_records,attrs_after='gold',show_progress=True,n_jobs=-1) candset_features_df.fillna(value=0, inplace=True) return candset_features_df
# spliting data into training and testing sets train_test = em.split_train_test(tbl_labeled, train_proportion=0.7) dev_set = train_test['train'] eval_set = train_test['test'] em.to_csv_metadata(dev_set, 'datasets/dev_set.csv') em.to_csv_metadata(eval_set, 'datasets/eval_set.csv') # myset = em.split_train_test(dev_set, train_proportion=0.9) # I_set = myset['train'] # J_set = myset['test'] # em.to_csv_metadata(I_set, 'datasets/I_set.csv') # em.to_csv_metadata(J_set, 'datasets/J_set.csv') # creating feature for matching match_t = em.get_tokenizers_for_matching() match_s = em.get_sim_funs_for_matching() atypes1 = em.get_attr_types(sampled_movies) atypes2 = em.get_attr_types(sampled_tracks) match_c = em.get_attr_corres(sampled_movies, sampled_tracks) match_f = em.get_features(sampled_movies, sampled_tracks, atypes1, atypes2, match_c, match_t, match_s) # generating feature vectors H = em.extract_feature_vecs(dev_set, feature_table=match_f, attrs_after='label', show_progress=False) # filling missing values in feature vectors H.fillna(value=0, inplace=True)
def run_magellan(train_set, valid_set, test_set, feature_combinations, classifiers, experiment_name, write_test_set_for_inspection=False): train_path = os.path.dirname(train_set) train_file = os.path.basename(train_set) test_path = os.path.dirname(test_set) test_file = os.path.basename(test_set) report_train_name = train_file.replace('.csv', '') report_test_name = test_file.replace('.csv', '') train_set_left = train_file.replace('pairs', 'left') train_set_right = train_file.replace('pairs', 'right') test_set_left = test_file.replace('pairs', 'left') test_set_right = test_file.replace('pairs', 'right') os.makedirs(os.path.dirname( '../../../reports/magellan/{}/'.format(experiment_name)), exist_ok=True) try: os.remove('../../../reports/magellan/{}/{}_{}.csv'.format( experiment_name, report_train_name, report_test_name)) except OSError: pass with open( '../../../reports/magellan/{}/{}_{}.csv'.format( experiment_name, report_train_name, report_test_name), "w") as f: f.write( 'feature#####model#####mean_train_score#####std_train_score#####mean_valid_score#####std_valid_score#####precision_test#####recall_test#####f1_test#####best_params#####train_time#####prediction_time#####feature_importance#####experiment_name#####train_set#####test_set\n' ) for run in range(1, 4): for feature_combination in feature_combinations: A_t = em.read_csv_metadata(train_path + '/' + train_set_left, key='mag_id') B_t = em.read_csv_metadata(train_path + '/' + train_set_right, key='mag_id') # Load the pre-labeled data S_t = em.read_csv_metadata(train_set, key='_id', ltable=A_t, rtable=B_t, fk_ltable='ltable_mag_id', fk_rtable='rtable_mag_id') A_gs = em.read_csv_metadata(test_path + '/' + test_set_left, key='mag_id') B_gs = em.read_csv_metadata(test_path + '/' + test_set_right, key='mag_id') # Load the pre-labeled data S_gs = em.read_csv_metadata(test_set, key='_id', ltable=A_gs, rtable=B_gs, fk_ltable='ltable_mag_id', fk_rtable='rtable_mag_id') A_t.fillna('', inplace=True) A_gs.fillna('', inplace=True) B_t.fillna('', inplace=True) B_gs.fillna('', inplace=True) S_t.fillna('', inplace=True) S_gs.fillna('', inplace=True) ## DIRTY FIX, CLEAN UP! if 'name' in A_t.columns: A_t["price"] = A_t["price"].replace(r'^\s*$', np.nan, regex=True) A_t["price"] = A_t["price"].astype('float64') A_gs["price"] = A_gs["price"].replace(r'^\s*$', np.nan, regex=True) A_gs["price"] = A_gs["price"].astype('float64') B_t["price"] = B_t["price"].replace(r'^\s*$', np.nan, regex=True) B_t["price"] = B_t["price"].astype('float64') B_gs["price"] = B_gs["price"].replace(r'^\s*$', np.nan, regex=True) B_gs["price"] = B_gs["price"].astype('float64') S_t["ltable_price"] = S_t["ltable_price"].replace(r'^\s*$', np.nan, regex=True) S_t["ltable_price"] = S_t["ltable_price"].astype('float64') S_t["rtable_price"] = S_t["rtable_price"].replace(r'^\s*$', np.nan, regex=True) S_t["rtable_price"] = S_t["rtable_price"].astype('float64') S_gs["ltable_price"] = S_gs["ltable_price"].replace(r'^\s*$', np.nan, regex=True) S_gs["ltable_price"] = S_gs["ltable_price"].astype('float64') S_gs["rtable_price"] = S_gs["rtable_price"].replace(r'^\s*$', np.nan, regex=True) S_gs["rtable_price"] = S_gs["rtable_price"].astype('float64') atypes1 = em.get_attr_types(A_t) atypes2 = em.get_attr_types(B_t) match_c = em.get_attr_corres(A_t, B_t) match_c['corres'] = [] # select attributes to compare for feature in feature_combination: match_c['corres'].append((feature, feature)) tok = em.get_tokenizers_for_matching() sim = em.get_sim_funs_for_matching() F_t = em.get_features(A_t, B_t, atypes1, atypes2, match_c, tok, sim) H_t = em.extract_feature_vecs(S_t, feature_table=F_t, attrs_after=['label', 'pair_id'], show_progress=False) H_gs = em.extract_feature_vecs(S_gs, feature_table=F_t, attrs_after='label', show_progress=False) H_t = H_t.fillna(-1) H_gs = H_gs.fillna(-1) validation_ids_df = pd.read_csv(valid_set) val_df = H_t[H_t['pair_id'].isin( validation_ids_df['pair_id'].values)] train_only_df = H_t[~H_t['pair_id']. isin(validation_ids_df['pair_id'].values)] train_only_df = train_only_df.drop(columns='pair_id') val_df = val_df.drop(columns='pair_id') train_only_df = train_only_df.sample(frac=1, random_state=42) pos_neg = H_t['label'].value_counts() pos_neg = round(pos_neg[0] / pos_neg[1]) train_ind = [] val_ind = [] for i in range(len(train_only_df) - 1): train_ind.append(-1) for i in range(len(val_df) - 1): val_ind.append(0) ps = PredefinedSplit(test_fold=np.concatenate((train_ind, val_ind))) train_df = pd.concat([train_only_df, val_df]) for k, v in classifiers.items(): classifier = v['clf'] if 'random_state' in classifier.get_params().keys(): classifier = classifier.set_params(**{'random_state': run}) # add pos_neg ratio to XGBoost params if k == 'XGBoost': v['params']['scale_pos_weight']: [1, pos_neg] model = RandomizedSearchCV(cv=ps, estimator=classifier, param_distributions=v['params'], random_state=42, n_jobs=4, scoring='f1', n_iter=500, pre_dispatch=8, return_train_score=True) feats_train = train_df.drop( ['_id', 'ltable_mag_id', 'rtable_mag_id', 'label'], axis=1) labels_train = train_df['label'] feats_gs = H_gs.drop( ['_id', 'ltable_mag_id', 'rtable_mag_id', 'label'], axis=1) labels_gs = H_gs['label'] try: model.fit(feats_train, labels_train) except ValueError: set_trace() parameters = model.best_params_ score_names = [ 'mean_train_score', 'std_train_score', 'mean_test_score', 'std_test_score' ] scores = {} score_string = '' for name in score_names: scores[name] = model.cv_results_[name][model.best_index_] score_string = score_string + name + ': ' + str( scores[name]) + ' ' feature_names = list(feats_train.columns) if k == 'LogisticRegression' or k == 'LinearSVC': most_important_features = model.best_estimator_.coef_ word_importance = zip(feature_names, most_important_features[0].tolist()) word_importance = sorted( word_importance, key=lambda importance: importance[1], reverse=True) if k == 'RandomForest' or k == 'DecisionTree': most_important_features = model.best_estimator_.feature_importances_ word_importance = zip(feature_names, most_important_features.tolist()) word_importance = sorted( word_importance, key=lambda importance: importance[1], reverse=True) if k == 'NaiveBayes': word_importance = '' if k == 'XGBoost': most_important_features = model.best_estimator_.feature_importances_ word_importance = zip(feature_names, most_important_features.tolist()) word_importance = sorted( word_importance, key=lambda importance: importance[1], reverse=True) if k == 'LogisticRegression': learner = LogisticRegression(random_state=run, solver='liblinear', **parameters) elif k == 'NaiveBayes': learner = GaussianNB() elif k == 'DecisionTree': learner = DecisionTreeClassifier(random_state=run, **parameters) elif k == 'LinearSVC': learner = LinearSVC(random_state=run, dual=False, **parameters) elif k == 'RandomForest': learner = RandomForestClassifier(random_state=run, n_jobs=4, **parameters) elif k == 'XGBoost': learner = xgb.XGBClassifier(random_state=run, n_jobs=4, **parameters) else: print('Learner is not a valid option') break model = learner feats_train = train_only_df.sample(frac=1, random_state=42) feats_train = train_only_df.drop( ['_id', 'ltable_mag_id', 'rtable_mag_id', 'label'], axis=1) labels_train = train_only_df['label'] start = time.time() model.fit(feats_train, labels_train) end = time.time() train_time = end - start start = time.time() preds_gs = model.predict(feats_gs) end = time.time() pred_time = end - start gs_report = classification_report(labels_gs, preds_gs, output_dict=True) feature_report = '+'.join(feature_combination) if write_test_set_for_inspection: out_path = '../../../data/processed/wdc-lspc/inspection/{}/magellan/'.format( experiment_name) os.makedirs(os.path.dirname(out_path), exist_ok=True) file_name = '_'.join([ os.path.basename(train_set), os.path.basename(test_set), k, feature_report ]) file_name = file_name.replace('.csv', '') file_name += f'_{run}.pkl.gz' test_inspection_df = S_gs.copy() if k == 'LinearSVC': proba_gs = model.decision_function(feats_gs).tolist() else: proba_gs = model.predict_proba(feats_gs).tolist() test_inspection_df['pred'] = preds_gs test_inspection_df['Class Prob'] = proba_gs test_inspection_df.to_pickle(out_path + file_name, compression='gzip') with open( '../../../reports/magellan/{}/{}_{}.csv'.format( experiment_name, report_train_name, report_test_name), "a") as f: f.write(feature_report + '#####' + k + '#####' + str(scores['mean_train_score']) + '#####' + str(scores['std_train_score']) + '#####' + str(scores['mean_test_score']) + '#####' + str(scores['std_test_score']) + '#####' + str(gs_report['1']['precision']) + '#####' + str(gs_report['1']['recall']) + '#####' + str(gs_report['1']['f1-score']) + '#####' + str(parameters) + '#####' + str(train_time) + '#####' + str(pred_time) + '#####' + str(word_importance[0:100]) + '#####' + experiment_name + '#####' + report_train_name + '#####' + report_test_name + '\n')
''' helper.py ''' import random from sklearn.ensemble import RandomForestClassifier from sklearn.tree import DecisionTreeClassifier import py_entitymatching as em tok_name2func = em.get_tokenizers_for_matching(q=[2, 3, 4, 5]) sim_name2func = em.get_sim_funs_for_matching() def get_features_in_random_forest(rf: RandomForestClassifier) -> list: rf_features = set() for tree in rf.estimators_: for f in tree.tree_.feature: if f != -2: rf_features.add(f) return list(rf_features) def get_features_in_decision_tree(tree: DecisionTreeClassifier) -> list: tree_features = set() for f in tree.tree_.feature: if f != -2: tree_features.add(f) return list(tree_features)