def trex_method(args, model, test_ndx, X_train, y_train, X_test, logger=None): """ Generate an training instance attributions for a test instance. """ # train surrogate model start = time.time() params = { 'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel } surrogate = trex.train_surrogate(model=model, surrogate=args.method, X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) train_time = time.time() - start # compute influential training instances on the test instance start = time.time() surrogate.compute_attributions(X_test[test_ndx]) test_time = time.time() - start # result object result = {'train_time': train_time, 'test_time': test_time} return result
def teknn_method(args, model, X_train, y_train, X_test, logger=None): """ Sort trainnig instance based on similarity density to the test instances. """ # train surrogate model params = util.get_selected_params(dataset=args.dataset, model=args.model, surrogate=args.method) train_label = y_train if 'og' in args.method else model.predict(X_train) surrogate = trex.train_surrogate(model=model, surrogate='knn', X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=None) # sort instances based on largest influence w.r.t. the predicted labels of the test set if args.start_pred == -1: attributions = surrogate.compute_attributions(X_test).sum(axis=0) # sort instances based on largest influence w.r.t. the given label else: attributions = surrogate.compute_attributions( X_test, start_pred=args.start_pred).sum(axis=0) # sort based by largest similarity-influence to the test set train_indices = np.argsort(attributions)[::-1] return train_indices
def teknn_method(args, model_noisy, y_train_noisy, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=None, frac_progress_update=0.1): """ Count impact by the number of times a training sample shows up in one another's neighborhood, this can be weighted by 1 / distance. """ # train surrogate model params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel} train_label = y_train_noisy if 'og' in args.method else model_noisy.predict(X_train) surrogate = trex.train_surrogate(model=model_noisy, surrogate=args.method.split('_')[0], X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # sort by instance log loss using the surrogate model if 'loss' in args.method: # display progress if logger: logger.info('\ncomputing KNN loss...') y_train_proba = surrogate.predict_proba(X_train)[:, 1] y_train_noisy_loss = util.instance_log_loss(y_train_noisy, y_train_proba) # negative log-likelihood train_indices = np.argsort(y_train_noisy_loss)[::-1] # descending, largest log loss first # sort by absolute value of instance weights else: # sort instances based on largest influence toward the predicted training labels attributions = surrogate.compute_attributions(X_train, logger=logger) attributions_sum = np.sum(attributions, axis=0) train_indices = np.argsort(attributions_sum)[::-1] # fix noisy instances result = fix_noisy_instances(train_indices, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=logger) return result
def trex_method(args, model, X_train, y_train, X_test, logger=None, frac_progress_update=0.1): """ Sort training instances by largest 'excitatory' influnce on the test set. """ # train surrogate model params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel} surrogate = trex.train_surrogate(model=model, surrogate='klr', X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # display status if logger: logger.info('\ncomputing influence of each training sample on the test set...') # sort instances with the larget influence on the predicted labels of the test set pred = model.predict(X_test) attributions = surrogate.pred_influence(X_test, pred) attributions_sum = np.sum(attributions, axis=0) train_indices = np.argsort(attributions_sum)[::-1] # display k most influential training samples if logger: k = 20 sim_s = surrogate.similarity(X_test)[0][train_indices] alpha_s = surrogate.get_alpha()[train_indices] attributions_sum_s = attributions_sum[train_indices] y_train_s = y_train[train_indices] train_info = list(zip(train_indices, attributions_sum_s, y_train_s, alpha_s, sim_s)) s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}' for ndx, atr, lab, alpha, sim in train_info[:k]: logger.info(s.format(ndx, lab, alpha, sim, atr)) return train_indices
def teknn_method(args, model, test_ndx, X_train, y_train, X_test, logger=None): """ KNN surrogate method that retrieves its k nearest neighbors as most influential. """ with timeout(seconds=args.max_time): try: # train surrogate model start = time.time() params = { 'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel } surrogate = trex.train_surrogate(model=model, surrogate=args.method, X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) train_time = time.time() - start except TimeoutError: if logger: logger.info('TEKNN fine-tuning exceeded!') exit(0) # retrieve k nearest neighbors as most influential to the test instance start = time.time() surrogate.compute_attributions(X_test[test_ndx]) test_time = time.time() - start # result object result = {'train_time': train_time, 'test_time': test_time} return result
def teknn_method(args, model, X_train, y_train, X_test, logger=None): """ Sort trainnig instance based on similarity density to the test instances. """ # train surrogate model params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel} surrogate = trex.train_surrogate(model=model, surrogate=args.method, X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # sort instances based on largest influence on predicted test labels attributions = surrogate.compute_attributions(X_test) attributions_sum = np.sum(attributions, axis=0) train_indices = np.argsort(attributions_sum)[::-1] return train_indices
def trex_method(args, model_noisy, y_train_noisy, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=None): """ Order by largest absolute values of the instance coefficients from the KLR or SVM surrogate model. """ # train surrogate model params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel} train_label = y_train_noisy if 'og' in args.method else model_noisy.predict(X_train) surrogate = trex.train_surrogate(model=model_noisy, surrogate=args.method.split('_')[0], X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # sort by instance log loss using the surrogate model if 'loss' in args.method: y_train_proba = surrogate.predict_proba(X_train)[:, 1] y_train_noisy_loss = util.instance_log_loss(y_train_noisy, y_train_proba) # negative log-likelihood train_indices = np.argsort(y_train_noisy_loss)[::-1] # descending, largest log loss first # sort by absolute value of instance weights else: train_indices = np.argsort(np.abs(surrogate.get_alpha()))[::-1] # fix noisy instances result = fix_noisy_instances(train_indices, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=logger) return result
def trex_method(args, model, X_train, y_train, X_test, out_dir, frac_remove, logger=None): """ Sort training instances by largest 'excitatory' influnce on the test set. There is a difference between 1) the trainnig instances with the largest excitatory (or inhibitory) attributions, and 2) the training instances with the largest influence on the PREDICTED labels (i.e. excitatory or inhibitory attributions w.r.t. the predicted label). """ # train surrogate model params = util.get_selected_params(dataset=args.dataset, model=args.model, surrogate=args.method) train_label = y_train if 'og' in args.method else model.predict(X_train) logger.info('\nparams: {}'.format(params)) surrogate = trex.train_surrogate(model=model, surrogate='klr', X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=None) # save alpha if frac_remove == 0.0: alpha_dict = {'alpha': surrogate.get_alpha(), 'y_train': y_train} np.save(os.path.join(out_dir, 'alpha.npy'.format(frac_remove)), alpha_dict) # display status if not logger: logger.info( '\ncomputing influence of each training sample on the test set...') # sort by sim if 'sim' in args.method: attributions = surrogate.similarity(X_test) # compute influence based on predicted labels if args.start_pred == -1: pred_label = model.predict(X_test) # put positive weight if similar instances have the same label as the predicted test label for i in range(X_test.shape[0]): attributions[i] = np.where(train_label == pred_label[i], attributions[i], attributions[i] * -1) # put positive weight if similar instances have the same label as the predetermined label else: for i in range(X_test.shape[0]): attributions[i] = np.where(train_label == args.start_pred, attributions[i], attributions[i] * -1) # sort indices by largest similarity-influence to the test or predetermined label train_indices = np.argsort(attributions.sum(axis=0))[::-1] # sort by alpha elif 'alpha' in args.method: alpha = surrogate.get_alpha() # sort by largest to smallest magnitude if args.start_pred == -1: train_indices = np.argsort(np.abs(alpha))[::-1] print(alpha[train_indices]) # sort by most pos.to most neg. if `start_pred` is 1, and vice versa if 0 else: train_indices = np.argsort( alpha) if args.start_pred == 0 else np.argsort(alpha)[::-1] # sort by alpha * sim else: # compute influence based on predicted labels if args.start_pred == -1: # sort instances with the larget influence on the predicted labels of the test set pred = model.predict(X_test) attributions = surrogate.pred_influence(X_test, pred).sum(axis=0) train_indices = np.argsort(attributions)[::-1] # compute influence based on a predetermined label else: # sort by most excitatory or inhibitory attributions = surrogate.compute_attributions(X_test).sum(axis=0) train_indices = np.argsort( attributions)[::-1] if args.start_pred == 1 else np.argsort( attributions) return train_indices
def experiment(args, out_dir, logger): # start timer begin = time.time() # get data data = util.get_data(args.dataset, data_dir=args.data_dir, preprocessing=args.preprocessing) X_train, X_test, y_train, y_test, feature, cat_indices = data # randomly select a subset of test instances to evaluate with rng = np.random.default_rng(args.rs) indices = rng.choice(X_test.shape[0], size=args.n_test, replace=False) X_test, y_test = X_test[indices], y_test[indices] logger.info('\ntrain instances: {:,}'.format(X_train.shape[0])) logger.info('test instances: {:,}'.format(X_test.shape[0])) logger.info('no. features: {:,}'.format(X_train.shape[1])) # get tree-ensemble model = util.get_model(args.model, n_estimators=args.n_estimators, max_depth=args.max_depth, random_state=args.rs, cat_indices=cat_indices) logger.info('\nno. trees: {:,}'.format(args.n_estimators)) logger.info('max depth: {}'.format(args.max_depth)) # train a tree ensemble start = time.time() model = model.fit(X_train, y_train) logger.info('\nfitting tree ensemble...{:.3f}s'.format(time.time() - start)) # model predictions model_proba = model.predict_proba(X_test)[:, 1] # used if no tuning is done on the surrogate model params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel} # train and predict with using a surrogate model start = time.time() surrogate = trex.train_surrogate(model=model, surrogate=args.surrogate, X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) train_time = time.time() - start # make predictions start = time.time() surrogate_proba = surrogate.predict_proba(X_test)[:, 1] logger.info('prediction time...{:.3f}s'.format(time.time() - start)) # save results result = {} result['model'] = args.model result['model_n_estimators'] = args.n_estimators result['model_max_depth'] = args.max_depth result['model_proba'] = model_proba result['surrogate'] = args.surrogate result['surrogate_n_features'] = surrogate.n_features_alt_ result['surrogate_tree_kernel'] = surrogate.tree_kernel_ result['surrogate_C'] = surrogate.C if hasattr(surrogate, 'C') else None result['surrogate_n_neighbors'] = surrogate.n_neighbors if hasattr(surrogate, 'n_neighbors') else None result['surrogate_train_time'] = train_time result['surrogate_proba'] = surrogate_proba result['total_time'] = time.time() - begin result['max_rss'] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss result['tune_frac'] = args.tune_frac result['pearson'] = pearsonr(model_proba, surrogate_proba)[0] result['spearman'] = spearmanr(model_proba, surrogate_proba)[0] result['mse'] = mean_squared_error(model_proba, surrogate_proba) np.save(os.path.join(out_dir, 'results.npy'), result) logger.info('\nresults:\n{}'.format(result)) # save scatter fig, ax = plt.subplots() ax.scatter(result['surrogate_proba'], result['model_proba']) ax.set_xlabel('Surrogate prob.') ax.set_ylabel('Tree-ensemble prob.') ax.set_xlim(0, 1) ax.set_ylim(0, 1) plt.savefig(os.path.join(out_dir, 'scatter.pdf')) logger.info('\nsaving results to {}/...'.format(os.path.join(out_dir)))
def trex_method(args, model, X_train, y_train, X_test, logger=None, frac_progress_update=0.1): """ Sort training instances by most excitatory or most inhibitory on the test set. """ # train surrogate model params = { 'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel } surrogate = trex.train_surrogate(model=model, surrogate=args.method, X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # display status if logger: logger.info( '\ncomputing influence of each training sample on the test set...') # sort instances by most excitatory or most inhibitory attributions = surrogate.compute_attributions(X_test) attributions_sum = np.sum(attributions, axis=0) # sort by most excitatory and inhibitory n_excitatory = len(np.where(attributions_sum > 0)[0]) n_inhibitory = len(np.where(attributions_sum < 0)[0]) excitatory_train_indices = np.argsort( attributions_sum)[::-1][:n_excitatory] inhibitory_train_indices = np.argsort(attributions_sum)[:n_inhibitory] # display attributions from the top k train instances if logger: k = 20 # display most excitatory training instances sim_s = surrogate.similarity(X_test)[0][excitatory_train_indices] alpha_s = surrogate.get_alpha()[excitatory_train_indices] attributions_sum_s = attributions_sum[excitatory_train_indices] y_train_s = y_train[excitatory_train_indices] train_info = list( zip(excitatory_train_indices, attributions_sum_s, y_train_s, alpha_s, sim_s)) s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}' logger.info('\nExcitatory training instances...') for ndx, atr, lab, alpha, sim in train_info[:k]: logger.info(s.format(ndx, lab, alpha, sim, atr)) # display most inhibitory training instances sim_s = surrogate.similarity(X_test)[0][inhibitory_train_indices] alpha_s = surrogate.get_alpha()[inhibitory_train_indices] attributions_sum_s = attributions_sum[inhibitory_train_indices] y_train_s = y_train[inhibitory_train_indices] train_info = list( zip(inhibitory_train_indices, attributions_sum_s, y_train_s, alpha_s, sim_s)) s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}' logger.info('\nInhibitory training instances...') for ndx, atr, lab, alpha, sim in train_info[:k]: logger.info(s.format(ndx, lab, alpha, sim, atr)) return excitatory_train_indices, inhibitory_train_indices
def experiment(args, logger, out_dir): # start timer begin = time.time() # create random number generator rng = np.random.default_rng(args.rs) # get data data = util.get_data(args.dataset, data_dir=args.data_dir, preprocessing=args.preprocessing, mismatch=True) X_train, X_test, y_train, y_test, feature, cat_indices = data # get tree-ensemble clf = util.get_model(args.model, n_estimators=args.n_estimators, max_depth=args.max_depth, random_state=args.rs, cat_indices=cat_indices) # display dataset statistics logger.info('\nno. train instances: {:,}'.format(X_train.shape[0])) logger.info('no. test instances: {:,}'.format(X_test.shape[0])) logger.info('no. features: {:,}'.format(X_train.shape[1])) logger.info('\npos. label % (train): {:.1f}%'.format( np.sum(y_train) / y_train.shape[0] * 100)) logger.info('pos. label % (test): {:.1f}%\n'.format( np.sum(y_test) / y_test.shape[0] * 100)) # train tree ensemble model = clone(clf).fit(X_train, y_train) util.performance(model, X_train, y_train, logger=logger, name='Train') util.performance(model, X_test, y_test, logger=logger, name='Test') # train surrogate model params = { 'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel } surrogate = trex.train_surrogate(model=model, surrogate='klr', X_train=X_train, y_train=y_train, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=logger) # extract predictions start = time.time() model_pred = model.predict(X_test) model_proba = model.predict_proba(X_test)[:, 1] logger.info('predicting...{:.3f}s'.format(time.time() - start)) # pick a test instance in which the person is <= 17 from the Adult dataset indices = np.where(X_test[:, args.age_ndx] <= 17)[0] test_ndx = rng.choice(indices) age_test_val = X_test[test_ndx][args.age_ndx] x_test = X_test[[test_ndx]] # show prediction for this test instance s = '\ntest: {}, actual: {}, proba.: {:.3f}, age: {:.0f}' logger.info( s.format(test_ndx, y_test[test_ndx], model_proba[test_ndx], age_test_val)) # sort based on similarity-influence if 'sim' in args.surrogate: # compute influence based on predicted labels attributions = surrogate.similarity(x_test) pred_label = model.predict(x_test) # put positive weight if similar instances have the same label as the predicted test label for i in range(x_test.shape[0]): attributions[i] = np.where(y_train == pred_label[i], attributions[i], attributions[i] * -1) attributions = attributions.sum(axis=0) attribution_indices = np.argsort(attributions)[::-1] # sort training instances by most influential to the predicted label else: attributions = surrogate.pred_influence(x_test, model_pred[[test_ndx]])[0] attribution_indices = np.argsort(attributions)[::-1] # sort training instances by most similar to the test instance sim = surrogate.similarity(x_test)[0] sim_indices = np.argsort(sim)[::-1] # get instance weights alpha = surrogate.get_alpha() # 1. show most influential training instances logger.info( '\nTop {:,} most influential samples to the predicted label...'.format( args.topk_inf)) show_instances(args, X_train, alpha, sim, attributions, y_train, attribution_indices, args.topk_inf, logger) # 2a. compute aggregate surrogate contribution of most influential train instances attr_all = np.sum(np.abs(attributions)) attr_pos = np.sum(np.where(attributions > 0, attributions, 0)) # sum of pos. attributions only attr_topk_inf = np.sum( np.abs(attributions[attribution_indices][:args.topk_inf])) attr_topk_inf_pct = attr_topk_inf / attr_all * 100 attr_topk_inf_pos_pct = attr_topk_inf / attr_pos * 100 # 2b. display aggregate attributions s1 = '\nattribution % of top {:,} infuential instances: {:.2f}%' s2 = 'attribution % of top {:,} infuential instances for pos. attributions: {:.2f}%' logger.info(s1.format(args.topk_inf, attr_topk_inf_pct)) logger.info(s2.format(args.topk_inf, attr_topk_inf_pos_pct)) # 3. compute change in predicted probability after REMOVING the most influential instances s = 'test: {}, actual: {}, proba.: {:.3f}, age: {:.0f}' logger.info('\nRemoving top {:,} influential instances..'.format( args.topk_inf)) new_X_train = np.delete(X_train, attribution_indices[:args.topk_inf], axis=0) new_y_train = np.delete(y_train, attribution_indices[:args.topk_inf]) new_model = clone(clf).fit(new_X_train, new_y_train) util.performance(model, new_X_train, new_y_train, logger=logger, name='Train') util.performance(model, X_test, y_test, logger=logger, name='Test') logger.info( s.format(test_ndx, y_test[test_ndx], new_model.predict_proba(X_test)[:, 1][test_ndx], age_test_val)) # 4a. compute change in predicted probability after FLIPPING the labels of the most influential instances s1 = 'test: {}, actual: {}, proba.: {:.3f}, age: {:.0f}' s2 = '\n{:,} out of the top {:,} most influential instances have age <= 17' logger.info( '\nFixing ONLY corrupted labels of the top {:,} influential instances..' .format(args.topk_inf)) new_X_train = X_train.copy() new_y_train = y_train.copy() # 4b. fixing a portion of the corrupted training instances temp_indices = np.where( X_train[attribution_indices][:args.topk_inf][:, args.age_ndx] <= 17)[0] age17_topk_inf_indices = attribution_indices[temp_indices] new_y_train[age17_topk_inf_indices] = 0 # 4c. fit new model and re-evaluate new_model = clone(clf).fit(new_X_train, new_y_train) util.performance(new_model, new_X_train, new_y_train, logger=logger, name='Train') util.performance(new_model, X_test, y_test, logger=logger, name='Test') logger.info( s1.format(test_ndx, y_test[test_ndx], new_model.predict_proba(X_test)[:, 1][test_ndx], age_test_val)) logger.info(s2.format(age17_topk_inf_indices.shape[0], args.topk_inf)) # 5. show most similar training instances logger.info( '\nTop {:,} most similar samples to the predicted label...'.format( args.topk_sim)) show_instances(args, X_train, alpha, sim, attributions, y_train, sim_indices, args.topk_sim, logger) # 6. of the most similar train instances, compute how many have age <= 17 num_age17_topk = np.where( X_train[sim_indices][:args.topk_sim][:, args.age_ndx] <= 17)[0].shape[0] logger.info( '\nTop {:,} most similar train instances with age <= 17: {:,}'.format( args.topk_sim, num_age17_topk)) # 7. plot similarity of train instances against their instance weights logger.info('\nplotting similarity vs. weights...') plot_similarity(args, alpha, sim, out_dir, logger) # 8. no. train instances with age <= 17 and an alpha coefficient < 0 neg_alpha_indices = np.where(attributions < 0)[0] num_age17_neg_alpha = np.where( X_train[neg_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0] logger.info('\nno. instances with age <= 17 and alpha < 0: {:,}'.format( num_age17_neg_alpha)) # 9. no. train instances with age <= 17 and an alpha coefficient >= 0 pos_alpha_indices = np.where(attributions >= 0)[0] num_age17_pos_alpha = np.where( X_train[pos_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0] logger.info('no. instances with age <= 17 and alpha >= 0: {:,}'.format( num_age17_pos_alpha)) # 10. no. train instances with age <= 17, similarity > thershold and an alpha coefficient < 0 s = 'no. instances with age <= 17, sim > {:.2f} and alpha >= 0: {:,}' neg_alpha_indices = np.where((attributions < 0) & (sim > args.sim_thresh))[0] num_age17_sim_neg_alpha = np.where( X_train[neg_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0] logger.info(s.format(args.sim_thresh, num_age17_sim_neg_alpha)) # 11. no. train instances with age <= 17, similarity > thershold and an alpha coefficient >= 0 s = 'no. instances with age <= 17, sim > {:.2f} and alpha >= 0: {:,}' pos_alpha_indices = np.where((attributions >= 0) & (sim > args.sim_thresh))[0] num_age17_sim_pos_alpha = np.where( X_train[pos_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0] logger.info(s.format(args.sim_thresh, num_age17_sim_pos_alpha)) # display total time logger.info('\ntotal time: {:.3f}s'.format(time.time() - begin))
def trex_method(args, model, X_train, y_train, X_test, out_dir, frac_remove, logger=None): """ Sort training instances by largest 'excitatory' influnce on the test set. There is a difference between 1) the trainnig instances with the largest excitatory (or inhibitory) attributions, and 2) the training instances with the largest influence on the PREDICTED labels (i.e. excitatory or inhibitory attributions w.r.t. the predicted label). """ # train surrogate model params = util.get_selected_params(dataset=args.dataset, model=args.model, surrogate=args.method) train_label = y_train if 'og' in args.method else model.predict(X_train) logger.info('\nparams: {}'.format(params)) surrogate = trex.train_surrogate(model=model, surrogate='klr', X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=None) # save alpha and sim alpha = surrogate.get_alpha() sim = surrogate.similarity(X_test) plot_trex(alpha, sim, y_train, out_dir) # display status if not logger: logger.info( '\ncomputing influence of each training sample on the test set...') # sort by sim if 'sim' in args.method: attributions = surrogate.similarity(X_test) # compute influence based on predicted labels model_pred = model.predict(X_test) for i in range(X_test.shape[0]): attributions[i] = np.where(train_label == model_pred[i], attributions[i], attributions[i] * -1) # sort indices by largest similarity-influence to the test or predetermined label train_indices = np.argsort(attributions.sum(axis=0))[::-1] # sort by alpha elif 'alpha' in args.method: alpha = surrogate.get_alpha() magnitude = np.abs(alpha) train_indices = np.argsort(magnitude)[::-1] # sort by largest avg. influence on the test set else: pred = model.predict(X_test) influence = surrogate.pred_influence(X_test, pred) influence = np.mean(influence, axis=0) train_indices = np.argsort(influence)[::-1] return train_indices
def trex_method(args, model_noisy, y_train_noisy, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=None, out_dir=None): """ Order by largest absolute values of the instance coefficients from the KLR or SVM surrogate model. """ # train surrogate model params = util.get_selected_params(dataset=args.dataset, model=args.model, surrogate=args.method) train_label = y_train_noisy if 'og' in args.method else model_noisy.predict( X_train) surrogate = trex.train_surrogate(model=model_noisy, surrogate='klr', X_train=X_train, y_train=train_label, val_frac=args.tune_frac, metric=args.metric, seed=args.rs, params=params, logger=None) # sort by instance log loss using the surrogate model if 'loss' in args.method: y_train_proba = surrogate.predict_proba(X_train)[:, 1] y_train_noisy_loss = util.instance_log_loss( y_train_noisy, y_train_proba) # negative log-likelihood train_indices = np.argsort( y_train_noisy_loss)[::-1] # descending, largest log loss first # sort by sim. or alpha else: logger.info('\nsorting train instances...') # sort by similarity if 'sim' in args.method: start = time.time() # prioritize largest absolute similarity density if 'abs' in args.method: similarity_density = np.zeros(0, ) n_chunk = int(X_train.shape[0] * 0.1) for i in range(0, X_train.shape[0], n_chunk): X_sub_sim = surrogate.similarity(X_train[i:i + n_chunk]) similarity_density_sub = np.sum(X_sub_sim, axis=1) similarity_density = np.concatenate( [similarity_density, similarity_density_sub]) elapsed = time.time() - start logger.info('{:.1f}%...{:.3f}s'.format( i / X_train.shape[0] * 100, elapsed)) similarity_density = np.abs(similarity_density) train_indices = np.argsort(similarity_density)[::-1] # sort train instances prioritizing largest negative similarity density else: similarity_density = np.zeros(0, ) n_chunk = int(X_train.shape[0] * 0.1) for i in range(0, X_train.shape[0], n_chunk): X_sub_sim = surrogate.similarity(X_train[i:i + n_chunk]) y_sub_mask = np.ones(X_sub_sim.shape) for j in range(y_sub_mask.shape[0]): y_sub_mask[j][np.where( y_train_noisy[j + i] != y_train_noisy)] = -1 X_sub_sim = X_sub_sim * y_sub_mask similarity_density_sub = np.sum(X_sub_sim, axis=1) similarity_density = np.concatenate( [similarity_density, similarity_density_sub]) elapsed = time.time() - start logger.info('{:.1f}%...{:.3f}s'.format( i / X_train.shape[0] * 100, elapsed)) train_indices = np.argsort(similarity_density) # plot |alpha| vs. similarity density if out_dir is not None: alpha = surrogate.get_alpha() alpha = np.abs(alpha) non_noisy_indices = np.setdiff1d(np.arange(y_train.shape[0]), noisy_indices) fig, ax = plt.subplots() ax.scatter(alpha[non_noisy_indices], similarity_density[non_noisy_indices], alpha=0.1, label='non-noisy', color='green') ax.scatter(alpha[noisy_indices], similarity_density[noisy_indices], alpha=0.1, label='noisy', color='red') ax.set_xlabel('alpha') ax.set_ylabel('similarity_density') ax.legend() plt.savefig(os.path.join(out_dir, 'alpha_sim.png')) elif 'alpha' in args.method: alpha = surrogate.get_alpha() magnitude = np.abs(alpha) train_indices = np.argsort(magnitude)[::-1] else: raise ValueError('unknown method {}'.format(args.method)) # fix noisy instances result = fix_noisy_instances(train_indices, noisy_indices, n_check, n_checkpoint, clf, X_train, y_train, X_test, y_test, acc_noisy, auc_noisy, logger=logger) return result