def trex_method(args, model, test_ndx, X_train, y_train, X_test, logger=None):
"""
Generate an training instance attributions for a test instance.
"""
# train surrogate model
start = time.time()
params = {
'C': args.C,
'n_neighbors': args.n_neighbors,
'tree_kernel': args.tree_kernel
}
surrogate = trex.train_surrogate(model=model,
surrogate=args.method,
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
train_time = time.time() - start
# compute influential training instances on the test instance
start = time.time()
surrogate.compute_attributions(X_test[test_ndx])
test_time = time.time() - start
# result object
result = {'train_time': train_time, 'test_time': test_time}
return result
def teknn_method(args, model, X_train, y_train, X_test, logger=None):
"""
Sort trainnig instance based on similarity density to the test instances.
"""
# train surrogate model
params = util.get_selected_params(dataset=args.dataset,
model=args.model,
surrogate=args.method)
train_label = y_train if 'og' in args.method else model.predict(X_train)
surrogate = trex.train_surrogate(model=model,
surrogate='knn',
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=None)
# sort instances based on largest influence w.r.t. the predicted labels of the test set
if args.start_pred == -1:
attributions = surrogate.compute_attributions(X_test).sum(axis=0)
# sort instances based on largest influence w.r.t. the given label
else:
attributions = surrogate.compute_attributions(
X_test, start_pred=args.start_pred).sum(axis=0)
# sort based by largest similarity-influence to the test set
train_indices = np.argsort(attributions)[::-1]
return train_indices
def teknn_method(args, model_noisy, y_train_noisy,
noisy_indices, n_check, n_checkpoint,
clf, X_train, y_train, X_test, y_test,
acc_noisy, auc_noisy, logger=None,
frac_progress_update=0.1):
"""
Count impact by the number of times a training sample shows up in
one another's neighborhood, this can be weighted by 1 / distance.
"""
# train surrogate model
params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel}
train_label = y_train_noisy if 'og' in args.method else model_noisy.predict(X_train)
surrogate = trex.train_surrogate(model=model_noisy,
surrogate=args.method.split('_')[0],
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# sort by instance log loss using the surrogate model
if 'loss' in args.method:
# display progress
if logger:
logger.info('\ncomputing KNN loss...')
y_train_proba = surrogate.predict_proba(X_train)[:, 1]
y_train_noisy_loss = util.instance_log_loss(y_train_noisy, y_train_proba) # negative log-likelihood
train_indices = np.argsort(y_train_noisy_loss)[::-1] # descending, largest log loss first
# sort by absolute value of instance weights
else:
# sort instances based on largest influence toward the predicted training labels
attributions = surrogate.compute_attributions(X_train, logger=logger)
attributions_sum = np.sum(attributions, axis=0)
train_indices = np.argsort(attributions_sum)[::-1]
# fix noisy instances
result = fix_noisy_instances(train_indices, noisy_indices, n_check, n_checkpoint,
clf, X_train, y_train, X_test, y_test,
acc_noisy, auc_noisy, logger=logger)
return result
def trex_method(args, model, X_train, y_train, X_test, logger=None,
frac_progress_update=0.1):
"""
Sort training instances by largest 'excitatory' influnce on the test set.
"""
# train surrogate model
params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel}
surrogate = trex.train_surrogate(model=model,
surrogate='klr',
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# display status
if logger:
logger.info('\ncomputing influence of each training sample on the test set...')
# sort instances with the larget influence on the predicted labels of the test set
pred = model.predict(X_test)
attributions = surrogate.pred_influence(X_test, pred)
attributions_sum = np.sum(attributions, axis=0)
train_indices = np.argsort(attributions_sum)[::-1]
# display k most influential training samples
if logger:
k = 20
sim_s = surrogate.similarity(X_test)[0][train_indices]
alpha_s = surrogate.get_alpha()[train_indices]
attributions_sum_s = attributions_sum[train_indices]
y_train_s = y_train[train_indices]
train_info = list(zip(train_indices, attributions_sum_s, y_train_s, alpha_s, sim_s))
s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}'
for ndx, atr, lab, alpha, sim in train_info[:k]:
logger.info(s.format(ndx, lab, alpha, sim, atr))
return train_indices
def teknn_method(args, model, test_ndx, X_train, y_train, X_test, logger=None):
"""
KNN surrogate method that retrieves its k nearest neighbors as most influential.
"""
with timeout(seconds=args.max_time):
try:
# train surrogate model
start = time.time()
params = {
'C': args.C,
'n_neighbors': args.n_neighbors,
'tree_kernel': args.tree_kernel
}
surrogate = trex.train_surrogate(model=model,
surrogate=args.method,
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
train_time = time.time() - start
except TimeoutError:
if logger:
logger.info('TEKNN fine-tuning exceeded!')
exit(0)
# retrieve k nearest neighbors as most influential to the test instance
start = time.time()
surrogate.compute_attributions(X_test[test_ndx])
test_time = time.time() - start
# result object
result = {'train_time': train_time, 'test_time': test_time}
return result
def teknn_method(args, model, X_train, y_train, X_test, logger=None):
"""
Sort trainnig instance based on similarity density to the test instances.
"""
# train surrogate model
params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel}
surrogate = trex.train_surrogate(model=model,
surrogate=args.method,
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# sort instances based on largest influence on predicted test labels
attributions = surrogate.compute_attributions(X_test)
attributions_sum = np.sum(attributions, axis=0)
train_indices = np.argsort(attributions_sum)[::-1]
return train_indices
def trex_method(args, model_noisy, y_train_noisy,
noisy_indices, n_check, n_checkpoint,
clf, X_train, y_train, X_test, y_test,
acc_noisy, auc_noisy, logger=None):
"""
Order by largest absolute values of the instance coefficients
from the KLR or SVM surrogate model.
"""
# train surrogate model
params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel}
train_label = y_train_noisy if 'og' in args.method else model_noisy.predict(X_train)
surrogate = trex.train_surrogate(model=model_noisy,
surrogate=args.method.split('_')[0],
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# sort by instance log loss using the surrogate model
if 'loss' in args.method:
y_train_proba = surrogate.predict_proba(X_train)[:, 1]
y_train_noisy_loss = util.instance_log_loss(y_train_noisy, y_train_proba) # negative log-likelihood
train_indices = np.argsort(y_train_noisy_loss)[::-1] # descending, largest log loss first
# sort by absolute value of instance weights
else:
train_indices = np.argsort(np.abs(surrogate.get_alpha()))[::-1]
# fix noisy instances
result = fix_noisy_instances(train_indices, noisy_indices, n_check, n_checkpoint,
clf, X_train, y_train, X_test, y_test,
acc_noisy, auc_noisy, logger=logger)
return result
def trex_method(args,
model,
X_train,
y_train,
X_test,
out_dir,
frac_remove,
logger=None):
"""
Sort training instances by largest 'excitatory' influnce on the test set.
There is a difference between
1) the trainnig instances with the largest excitatory (or inhibitory) attributions, and
2) the training instances with the largest influence on the PREDICTED labels
(i.e. excitatory or inhibitory attributions w.r.t. the predicted label).
"""
# train surrogate model
params = util.get_selected_params(dataset=args.dataset,
model=args.model,
surrogate=args.method)
train_label = y_train if 'og' in args.method else model.predict(X_train)
logger.info('\nparams: {}'.format(params))
surrogate = trex.train_surrogate(model=model,
surrogate='klr',
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=None)
# save alpha
if frac_remove == 0.0:
alpha_dict = {'alpha': surrogate.get_alpha(), 'y_train': y_train}
np.save(os.path.join(out_dir, 'alpha.npy'.format(frac_remove)),
alpha_dict)
# display status
if not logger:
logger.info(
'\ncomputing influence of each training sample on the test set...')
# sort by sim
if 'sim' in args.method:
attributions = surrogate.similarity(X_test)
# compute influence based on predicted labels
if args.start_pred == -1:
pred_label = model.predict(X_test)
# put positive weight if similar instances have the same label as the predicted test label
for i in range(X_test.shape[0]):
attributions[i] = np.where(train_label == pred_label[i],
attributions[i],
attributions[i] * -1)
# put positive weight if similar instances have the same label as the predetermined label
else:
for i in range(X_test.shape[0]):
attributions[i] = np.where(train_label == args.start_pred,
attributions[i],
attributions[i] * -1)
# sort indices by largest similarity-influence to the test or predetermined label
train_indices = np.argsort(attributions.sum(axis=0))[::-1]
# sort by alpha
elif 'alpha' in args.method:
alpha = surrogate.get_alpha()
# sort by largest to smallest magnitude
if args.start_pred == -1:
train_indices = np.argsort(np.abs(alpha))[::-1]
print(alpha[train_indices])
# sort by most pos.to most neg. if `start_pred` is 1, and vice versa if 0
else:
train_indices = np.argsort(
alpha) if args.start_pred == 0 else np.argsort(alpha)[::-1]
# sort by alpha * sim
else:
# compute influence based on predicted labels
if args.start_pred == -1:
# sort instances with the larget influence on the predicted labels of the test set
pred = model.predict(X_test)
attributions = surrogate.pred_influence(X_test, pred).sum(axis=0)
train_indices = np.argsort(attributions)[::-1]
# compute influence based on a predetermined label
else:
# sort by most excitatory or inhibitory
attributions = surrogate.compute_attributions(X_test).sum(axis=0)
train_indices = np.argsort(
attributions)[::-1] if args.start_pred == 1 else np.argsort(
attributions)
return train_indices
def experiment(args, out_dir, logger):
# start timer
begin = time.time()
# get data
data = util.get_data(args.dataset,
data_dir=args.data_dir,
preprocessing=args.preprocessing)
X_train, X_test, y_train, y_test, feature, cat_indices = data
# randomly select a subset of test instances to evaluate with
rng = np.random.default_rng(args.rs)
indices = rng.choice(X_test.shape[0], size=args.n_test, replace=False)
X_test, y_test = X_test[indices], y_test[indices]
logger.info('\ntrain instances: {:,}'.format(X_train.shape[0]))
logger.info('test instances: {:,}'.format(X_test.shape[0]))
logger.info('no. features: {:,}'.format(X_train.shape[1]))
# get tree-ensemble
model = util.get_model(args.model,
n_estimators=args.n_estimators,
max_depth=args.max_depth,
random_state=args.rs,
cat_indices=cat_indices)
logger.info('\nno. trees: {:,}'.format(args.n_estimators))
logger.info('max depth: {}'.format(args.max_depth))
# train a tree ensemble
start = time.time()
model = model.fit(X_train, y_train)
logger.info('\nfitting tree ensemble...{:.3f}s'.format(time.time() - start))
# model predictions
model_proba = model.predict_proba(X_test)[:, 1]
# used if no tuning is done on the surrogate model
params = {'C': args.C, 'n_neighbors': args.n_neighbors, 'tree_kernel': args.tree_kernel}
# train and predict with using a surrogate model
start = time.time()
surrogate = trex.train_surrogate(model=model,
surrogate=args.surrogate,
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
train_time = time.time() - start
# make predictions
start = time.time()
surrogate_proba = surrogate.predict_proba(X_test)[:, 1]
logger.info('prediction time...{:.3f}s'.format(time.time() - start))
# save results
result = {}
result['model'] = args.model
result['model_n_estimators'] = args.n_estimators
result['model_max_depth'] = args.max_depth
result['model_proba'] = model_proba
result['surrogate'] = args.surrogate
result['surrogate_n_features'] = surrogate.n_features_alt_
result['surrogate_tree_kernel'] = surrogate.tree_kernel_
result['surrogate_C'] = surrogate.C if hasattr(surrogate, 'C') else None
result['surrogate_n_neighbors'] = surrogate.n_neighbors if hasattr(surrogate, 'n_neighbors') else None
result['surrogate_train_time'] = train_time
result['surrogate_proba'] = surrogate_proba
result['total_time'] = time.time() - begin
result['max_rss'] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
result['tune_frac'] = args.tune_frac
result['pearson'] = pearsonr(model_proba, surrogate_proba)[0]
result['spearman'] = spearmanr(model_proba, surrogate_proba)[0]
result['mse'] = mean_squared_error(model_proba, surrogate_proba)
np.save(os.path.join(out_dir, 'results.npy'), result)
logger.info('\nresults:\n{}'.format(result))
# save scatter
fig, ax = plt.subplots()
ax.scatter(result['surrogate_proba'], result['model_proba'])
ax.set_xlabel('Surrogate prob.')
ax.set_ylabel('Tree-ensemble prob.')
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
plt.savefig(os.path.join(out_dir, 'scatter.pdf'))
logger.info('\nsaving results to {}/...'.format(os.path.join(out_dir)))
def trex_method(args,
model,
X_train,
y_train,
X_test,
logger=None,
frac_progress_update=0.1):
"""
Sort training instances by most excitatory or most inhibitory on the test set.
"""
# train surrogate model
params = {
'C': args.C,
'n_neighbors': args.n_neighbors,
'tree_kernel': args.tree_kernel
}
surrogate = trex.train_surrogate(model=model,
surrogate=args.method,
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# display status
if logger:
logger.info(
'\ncomputing influence of each training sample on the test set...')
# sort instances by most excitatory or most inhibitory
attributions = surrogate.compute_attributions(X_test)
attributions_sum = np.sum(attributions, axis=0)
# sort by most excitatory and inhibitory
n_excitatory = len(np.where(attributions_sum > 0)[0])
n_inhibitory = len(np.where(attributions_sum < 0)[0])
excitatory_train_indices = np.argsort(
attributions_sum)[::-1][:n_excitatory]
inhibitory_train_indices = np.argsort(attributions_sum)[:n_inhibitory]
# display attributions from the top k train instances
if logger:
k = 20
# display most excitatory training instances
sim_s = surrogate.similarity(X_test)[0][excitatory_train_indices]
alpha_s = surrogate.get_alpha()[excitatory_train_indices]
attributions_sum_s = attributions_sum[excitatory_train_indices]
y_train_s = y_train[excitatory_train_indices]
train_info = list(
zip(excitatory_train_indices, attributions_sum_s, y_train_s,
alpha_s, sim_s))
s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}'
logger.info('\nExcitatory training instances...')
for ndx, atr, lab, alpha, sim in train_info[:k]:
logger.info(s.format(ndx, lab, alpha, sim, atr))
# display most inhibitory training instances
sim_s = surrogate.similarity(X_test)[0][inhibitory_train_indices]
alpha_s = surrogate.get_alpha()[inhibitory_train_indices]
attributions_sum_s = attributions_sum[inhibitory_train_indices]
y_train_s = y_train[inhibitory_train_indices]
train_info = list(
zip(inhibitory_train_indices, attributions_sum_s, y_train_s,
alpha_s, sim_s))
s = '[{:5}] label: {}, alpha: {:.3f}, sim: {:.3f} attribution sum: {:.3f}'
logger.info('\nInhibitory training instances...')
for ndx, atr, lab, alpha, sim in train_info[:k]:
logger.info(s.format(ndx, lab, alpha, sim, atr))
return excitatory_train_indices, inhibitory_train_indices
def experiment(args, logger, out_dir):
# start timer
begin = time.time()
# create random number generator
rng = np.random.default_rng(args.rs)
# get data
data = util.get_data(args.dataset,
data_dir=args.data_dir,
preprocessing=args.preprocessing,
mismatch=True)
X_train, X_test, y_train, y_test, feature, cat_indices = data
# get tree-ensemble
clf = util.get_model(args.model,
n_estimators=args.n_estimators,
max_depth=args.max_depth,
random_state=args.rs,
cat_indices=cat_indices)
# display dataset statistics
logger.info('\nno. train instances: {:,}'.format(X_train.shape[0]))
logger.info('no. test instances: {:,}'.format(X_test.shape[0]))
logger.info('no. features: {:,}'.format(X_train.shape[1]))
logger.info('\npos. label % (train): {:.1f}%'.format(
np.sum(y_train) / y_train.shape[0] * 100))
logger.info('pos. label % (test): {:.1f}%\n'.format(
np.sum(y_test) / y_test.shape[0] * 100))
# train tree ensemble
model = clone(clf).fit(X_train, y_train)
util.performance(model, X_train, y_train, logger=logger, name='Train')
util.performance(model, X_test, y_test, logger=logger, name='Test')
# train surrogate model
params = {
'C': args.C,
'n_neighbors': args.n_neighbors,
'tree_kernel': args.tree_kernel
}
surrogate = trex.train_surrogate(model=model,
surrogate='klr',
X_train=X_train,
y_train=y_train,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=logger)
# extract predictions
start = time.time()
model_pred = model.predict(X_test)
model_proba = model.predict_proba(X_test)[:, 1]
logger.info('predicting...{:.3f}s'.format(time.time() - start))
# pick a test instance in which the person is <= 17 from the Adult dataset
indices = np.where(X_test[:, args.age_ndx] <= 17)[0]
test_ndx = rng.choice(indices)
age_test_val = X_test[test_ndx][args.age_ndx]
x_test = X_test[[test_ndx]]
# show prediction for this test instance
s = '\ntest: {}, actual: {}, proba.: {:.3f}, age: {:.0f}'
logger.info(
s.format(test_ndx, y_test[test_ndx], model_proba[test_ndx],
age_test_val))
# sort based on similarity-influence
if 'sim' in args.surrogate:
# compute influence based on predicted labels
attributions = surrogate.similarity(x_test)
pred_label = model.predict(x_test)
# put positive weight if similar instances have the same label as the predicted test label
for i in range(x_test.shape[0]):
attributions[i] = np.where(y_train == pred_label[i],
attributions[i], attributions[i] * -1)
attributions = attributions.sum(axis=0)
attribution_indices = np.argsort(attributions)[::-1]
# sort training instances by most influential to the predicted label
else:
attributions = surrogate.pred_influence(x_test,
model_pred[[test_ndx]])[0]
attribution_indices = np.argsort(attributions)[::-1]
# sort training instances by most similar to the test instance
sim = surrogate.similarity(x_test)[0]
sim_indices = np.argsort(sim)[::-1]
# get instance weights
alpha = surrogate.get_alpha()
# 1. show most influential training instances
logger.info(
'\nTop {:,} most influential samples to the predicted label...'.format(
args.topk_inf))
show_instances(args, X_train, alpha, sim, attributions, y_train,
attribution_indices, args.topk_inf, logger)
# 2a. compute aggregate surrogate contribution of most influential train instances
attr_all = np.sum(np.abs(attributions))
attr_pos = np.sum(np.where(attributions > 0, attributions,
0)) # sum of pos. attributions only
attr_topk_inf = np.sum(
np.abs(attributions[attribution_indices][:args.topk_inf]))
attr_topk_inf_pct = attr_topk_inf / attr_all * 100
attr_topk_inf_pos_pct = attr_topk_inf / attr_pos * 100
# 2b. display aggregate attributions
s1 = '\nattribution % of top {:,} infuential instances: {:.2f}%'
s2 = 'attribution % of top {:,} infuential instances for pos. attributions: {:.2f}%'
logger.info(s1.format(args.topk_inf, attr_topk_inf_pct))
logger.info(s2.format(args.topk_inf, attr_topk_inf_pos_pct))
# 3. compute change in predicted probability after REMOVING the most influential instances
s = 'test: {}, actual: {}, proba.: {:.3f}, age: {:.0f}'
logger.info('\nRemoving top {:,} influential instances..'.format(
args.topk_inf))
new_X_train = np.delete(X_train,
attribution_indices[:args.topk_inf],
axis=0)
new_y_train = np.delete(y_train, attribution_indices[:args.topk_inf])
new_model = clone(clf).fit(new_X_train, new_y_train)
util.performance(model,
new_X_train,
new_y_train,
logger=logger,
name='Train')
util.performance(model, X_test, y_test, logger=logger, name='Test')
logger.info(
s.format(test_ndx, y_test[test_ndx],
new_model.predict_proba(X_test)[:, 1][test_ndx],
age_test_val))
# 4a. compute change in predicted probability after FLIPPING the labels of the most influential instances
s1 = 'test: {}, actual: {}, proba.: {:.3f}, age: {:.0f}'
s2 = '\n{:,} out of the top {:,} most influential instances have age <= 17'
logger.info(
'\nFixing ONLY corrupted labels of the top {:,} influential instances..'
.format(args.topk_inf))
new_X_train = X_train.copy()
new_y_train = y_train.copy()
# 4b. fixing a portion of the corrupted training instances
temp_indices = np.where(
X_train[attribution_indices][:args.topk_inf][:, args.age_ndx] <= 17)[0]
age17_topk_inf_indices = attribution_indices[temp_indices]
new_y_train[age17_topk_inf_indices] = 0
# 4c. fit new model and re-evaluate
new_model = clone(clf).fit(new_X_train, new_y_train)
util.performance(new_model,
new_X_train,
new_y_train,
logger=logger,
name='Train')
util.performance(new_model, X_test, y_test, logger=logger, name='Test')
logger.info(
s1.format(test_ndx, y_test[test_ndx],
new_model.predict_proba(X_test)[:, 1][test_ndx],
age_test_val))
logger.info(s2.format(age17_topk_inf_indices.shape[0], args.topk_inf))
# 5. show most similar training instances
logger.info(
'\nTop {:,} most similar samples to the predicted label...'.format(
args.topk_sim))
show_instances(args, X_train, alpha, sim, attributions, y_train,
sim_indices, args.topk_sim, logger)
# 6. of the most similar train instances, compute how many have age <= 17
num_age17_topk = np.where(
X_train[sim_indices][:args.topk_sim][:,
args.age_ndx] <= 17)[0].shape[0]
logger.info(
'\nTop {:,} most similar train instances with age <= 17: {:,}'.format(
args.topk_sim, num_age17_topk))
# 7. plot similarity of train instances against their instance weights
logger.info('\nplotting similarity vs. weights...')
plot_similarity(args, alpha, sim, out_dir, logger)
# 8. no. train instances with age <= 17 and an alpha coefficient < 0
neg_alpha_indices = np.where(attributions < 0)[0]
num_age17_neg_alpha = np.where(
X_train[neg_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0]
logger.info('\nno. instances with age <= 17 and alpha < 0: {:,}'.format(
num_age17_neg_alpha))
# 9. no. train instances with age <= 17 and an alpha coefficient >= 0
pos_alpha_indices = np.where(attributions >= 0)[0]
num_age17_pos_alpha = np.where(
X_train[pos_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0]
logger.info('no. instances with age <= 17 and alpha >= 0: {:,}'.format(
num_age17_pos_alpha))
# 10. no. train instances with age <= 17, similarity > thershold and an alpha coefficient < 0
s = 'no. instances with age <= 17, sim > {:.2f} and alpha >= 0: {:,}'
neg_alpha_indices = np.where((attributions < 0)
& (sim > args.sim_thresh))[0]
num_age17_sim_neg_alpha = np.where(
X_train[neg_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0]
logger.info(s.format(args.sim_thresh, num_age17_sim_neg_alpha))
# 11. no. train instances with age <= 17, similarity > thershold and an alpha coefficient >= 0
s = 'no. instances with age <= 17, sim > {:.2f} and alpha >= 0: {:,}'
pos_alpha_indices = np.where((attributions >= 0)
& (sim > args.sim_thresh))[0]
num_age17_sim_pos_alpha = np.where(
X_train[pos_alpha_indices][:, args.age_ndx] <= 17)[0].shape[0]
logger.info(s.format(args.sim_thresh, num_age17_sim_pos_alpha))
# display total time
logger.info('\ntotal time: {:.3f}s'.format(time.time() - begin))
def trex_method(args,
model,
X_train,
y_train,
X_test,
out_dir,
frac_remove,
logger=None):
"""
Sort training instances by largest 'excitatory' influnce on the test set.
There is a difference between
1) the trainnig instances with the largest excitatory (or inhibitory) attributions, and
2) the training instances with the largest influence on the PREDICTED labels
(i.e. excitatory or inhibitory attributions w.r.t. the predicted label).
"""
# train surrogate model
params = util.get_selected_params(dataset=args.dataset,
model=args.model,
surrogate=args.method)
train_label = y_train if 'og' in args.method else model.predict(X_train)
logger.info('\nparams: {}'.format(params))
surrogate = trex.train_surrogate(model=model,
surrogate='klr',
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=None)
# save alpha and sim
alpha = surrogate.get_alpha()
sim = surrogate.similarity(X_test)
plot_trex(alpha, sim, y_train, out_dir)
# display status
if not logger:
logger.info(
'\ncomputing influence of each training sample on the test set...')
# sort by sim
if 'sim' in args.method:
attributions = surrogate.similarity(X_test)
# compute influence based on predicted labels
model_pred = model.predict(X_test)
for i in range(X_test.shape[0]):
attributions[i] = np.where(train_label == model_pred[i],
attributions[i], attributions[i] * -1)
# sort indices by largest similarity-influence to the test or predetermined label
train_indices = np.argsort(attributions.sum(axis=0))[::-1]
# sort by alpha
elif 'alpha' in args.method:
alpha = surrogate.get_alpha()
magnitude = np.abs(alpha)
train_indices = np.argsort(magnitude)[::-1]
# sort by largest avg. influence on the test set
else:
pred = model.predict(X_test)
influence = surrogate.pred_influence(X_test, pred)
influence = np.mean(influence, axis=0)
train_indices = np.argsort(influence)[::-1]
return train_indices
def trex_method(args,
model_noisy,
y_train_noisy,
noisy_indices,
n_check,
n_checkpoint,
clf,
X_train,
y_train,
X_test,
y_test,
acc_noisy,
auc_noisy,
logger=None,
out_dir=None):
"""
Order by largest absolute values of the instance coefficients
from the KLR or SVM surrogate model.
"""
# train surrogate model
params = util.get_selected_params(dataset=args.dataset,
model=args.model,
surrogate=args.method)
train_label = y_train_noisy if 'og' in args.method else model_noisy.predict(
X_train)
surrogate = trex.train_surrogate(model=model_noisy,
surrogate='klr',
X_train=X_train,
y_train=train_label,
val_frac=args.tune_frac,
metric=args.metric,
seed=args.rs,
params=params,
logger=None)
# sort by instance log loss using the surrogate model
if 'loss' in args.method:
y_train_proba = surrogate.predict_proba(X_train)[:, 1]
y_train_noisy_loss = util.instance_log_loss(
y_train_noisy, y_train_proba) # negative log-likelihood
train_indices = np.argsort(
y_train_noisy_loss)[::-1] # descending, largest log loss first
# sort by sim. or alpha
else:
logger.info('\nsorting train instances...')
# sort by similarity
if 'sim' in args.method:
start = time.time()
# prioritize largest absolute similarity density
if 'abs' in args.method:
similarity_density = np.zeros(0, )
n_chunk = int(X_train.shape[0] * 0.1)
for i in range(0, X_train.shape[0], n_chunk):
X_sub_sim = surrogate.similarity(X_train[i:i + n_chunk])
similarity_density_sub = np.sum(X_sub_sim, axis=1)
similarity_density = np.concatenate(
[similarity_density, similarity_density_sub])
elapsed = time.time() - start
logger.info('{:.1f}%...{:.3f}s'.format(
i / X_train.shape[0] * 100, elapsed))
similarity_density = np.abs(similarity_density)
train_indices = np.argsort(similarity_density)[::-1]
# sort train instances prioritizing largest negative similarity density
else:
similarity_density = np.zeros(0, )
n_chunk = int(X_train.shape[0] * 0.1)
for i in range(0, X_train.shape[0], n_chunk):
X_sub_sim = surrogate.similarity(X_train[i:i + n_chunk])
y_sub_mask = np.ones(X_sub_sim.shape)
for j in range(y_sub_mask.shape[0]):
y_sub_mask[j][np.where(
y_train_noisy[j + i] != y_train_noisy)] = -1
X_sub_sim = X_sub_sim * y_sub_mask
similarity_density_sub = np.sum(X_sub_sim, axis=1)
similarity_density = np.concatenate(
[similarity_density, similarity_density_sub])
elapsed = time.time() - start
logger.info('{:.1f}%...{:.3f}s'.format(
i / X_train.shape[0] * 100, elapsed))
train_indices = np.argsort(similarity_density)
# plot |alpha| vs. similarity density
if out_dir is not None:
alpha = surrogate.get_alpha()
alpha = np.abs(alpha)
non_noisy_indices = np.setdiff1d(np.arange(y_train.shape[0]),
noisy_indices)
fig, ax = plt.subplots()
ax.scatter(alpha[non_noisy_indices],
similarity_density[non_noisy_indices],
alpha=0.1,
label='non-noisy',
color='green')
ax.scatter(alpha[noisy_indices],
similarity_density[noisy_indices],
alpha=0.1,
label='noisy',
color='red')
ax.set_xlabel('alpha')
ax.set_ylabel('similarity_density')
ax.legend()
plt.savefig(os.path.join(out_dir, 'alpha_sim.png'))
elif 'alpha' in args.method:
alpha = surrogate.get_alpha()
magnitude = np.abs(alpha)
train_indices = np.argsort(magnitude)[::-1]
else:
raise ValueError('unknown method {}'.format(args.method))
# fix noisy instances
result = fix_noisy_instances(train_indices,
noisy_indices,
n_check,
n_checkpoint,
clf,
X_train,
y_train,
X_test,
y_test,
acc_noisy,
auc_noisy,
logger=logger)
return result
