def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.info('print this')
params = FLAGS.flag_values_dict()
tf.set_random_seed(params['seed'])
params['results_dir'] = utils.make_subdir(params['results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
itr_train, itr_valid, itr_test = dataset_utils.load_dset_supervised()
conv_dims = [int(x) for x in params['conv_dims'].split(',')]
conv_sizes = [int(x) for x in params['conv_sizes'].split(',')]
dense_sizes = [int(x) for x in params['dense_sizes'].split(',')]
clf = CNN(conv_dims, conv_sizes, dense_sizes, params['n_classes'])
params['n_layers'] = len(conv_dims)
train_cnn.train_classifier(clf, itr_train, itr_valid, params)
train_cnn.test_classifier(clf, itr_test, params, 'test')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.info('print this')
params = FLAGS.flag_values_dict()
plt.rcParams['savefig.format'] = params['mpl_format']
params['results_dir'] = utils.make_subdir(params['results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['sampledir'] = utils.make_subdir(params['figdir'], 'samples')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
ood_classes = [int(x) for x in params['ood_classes'].split(',')]
# assume we train on all non-OOD classes
n_classes = 10
all_classes = range(n_classes)
ind_classes = [x for x in all_classes if x not in ood_classes]
(itr_train, itr_valid, itr_test,
itr_test_ood) = dataset_utils.load_dset_ood_unsupervised(
ind_classes, ood_classes)
conv_dims = [int(x) for x in params['conv_dims'].split(',')]
conv_sizes = [int(x) for x in params['conv_sizes'].split(',')]
vae = VAE(conv_dims, conv_sizes)
run_vae_mnist.train_vae(vae, itr_train, itr_valid, params)
run_vae_mnist.test_vae(vae, itr_test, params)
params['tensordir'] = utils.make_subdir(params['results_dir'],
'ood_tensors')
run_vae_mnist.test_vae(vae, itr_test_ood, params)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
params = FLAGS.flag_values_dict()
tf.set_random_seed(params['seed'])
params['results_dir'] = utils.make_subdir(params['training_results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
conv_dims = [int(x) for x in params['conv_dims'].split(',')]
conv_sizes = [int(x) for x in params['conv_sizes'].split(',')]
dense_sizes = [int(x) for x in params['dense_sizes'].split(',')]
params['n_layers'] = len(conv_dims)
clf = classifier.CNN(conv_dims,
conv_sizes,
dense_sizes,
params['n_classes'],
onehot=True)
utils.checkpoint_model(clf, params['ckptdir'], 'initmodel')
itr_train, itr_valid, itr_test = dataset_utils.load_dset_supervised_onehot(
)
train_cnn.train_classifier(clf, itr_train, itr_valid, params)
train_cnn.test_classifier(clf, itr_test, params, 'test')
train_data = utils.aggregate_batches(itr_train, 1000,
['train_x_infl', 'train_y_infl'])
validation_data = utils.aggregate_batches(itr_valid, 1000,
['valid_x_infl', 'valid_y_infl'])
test_data = utils.aggregate_batches(itr_test, 1000,
['test_x_infl', 'test_y_infl'])
utils.save_tensors(
train_data.items() + validation_data.items() + test_data.items(),
params['tensordir'])
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.info('print this')
params = FLAGS.flag_values_dict()
tf.set_random_seed(params['seed'])
plt.rcParams['savefig.format'] = params['mpl_format']
params['results_dir'] = utils.make_subdir(params['results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['sampledir'] = utils.make_subdir(params['figdir'], 'samples')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
itr_train, itr_valid, itr_test = dataset_utils.load_dset_unsupervised()
conv_dims = [int(x) for x in params['conv_dims'].split(',')]
conv_sizes = [int(x) for x in params['conv_sizes'].split(',')]
vae = VAE(conv_dims, conv_sizes)
train_vae(vae, itr_train, itr_valid, params)
test_vae(vae, itr_test, params)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
params = FLAGS.flag_values_dict()
tf.set_random_seed(params['seed'])
params['results_dir'] = utils.make_subdir(params['results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
ood_classes = [int(x) for x in params['ood_classes'].split(',')]
# We assume we train on all non-OOD classes.
all_classes = range(params['n_classes'])
ind_classes = [x for x in all_classes if x not in ood_classes]
(itr_train, itr_valid, itr_test,
itr_test_ood) = dataset_utils.load_dset_ood_supervised(
ind_classes, ood_classes)
conv_dims = [int(x) for x in params['conv_dims'].split(',')]
conv_sizes = [int(x) for x in params['conv_sizes'].split(',')]
clf = CNN(conv_dims, conv_sizes, params['n_classes'])
params['n_layers'] = len(conv_dims)
train_cnn.train_classifier(clf, itr_train, itr_valid, params)
train_cnn.test_classifier(clf, itr_test, params, 'test')
params['tensordir'] = utils.make_subdir(params['results_dir'],
'train_tensors')
train_cnn.test_classifier(clf, itr_train, params, 'train')
params['tensordir'] = utils.make_subdir(params['results_dir'],
'ood_tensors')
train_cnn.test_classifier(clf, itr_test_ood, params, 'ood')
def calculate_influence_ood(params):
"""Calculates influence functions for pre-trained model with OOD classes.
Args:
params (dict): contains a number of params - as loaded from flags.
Should contain:
seed (int) - random seed for Tensorflow and Numpy initialization.
training_results_dir (str) - parent directory of the pre-trained model.
clf_name (str) - the name of the pre-trained model's directory.
n_test_infl (int) - number of examples to run influence functions for.
start_ix_test_infl (int) - index to start loading examples from.
cg_maxiter (int) - max number of iterations for conjugate gradient.
squared (bool) - whether to calculate squared Hessian directly.
tol (float) - tolerance for conjugate gradient.
lam (float) - L2 regularization amount for Hessian.
hvp_samples (int) - number of samples to take in HVP estimation.
output_dir (str) - where results should be written - defaults to
training_results_dir/clf_name/influence_results.
tname (str) - extra string to add to saved tensor names; can be ''.
preloaded_model (model or None) - if None, we should load the model
ourselves. Otherwise, preloaded_model is the model we are interested in.
preloaded_itr (Iterator or None) - if None, load the data iterator
ourselves; otherwise, use preloaded_itr as the data iterator.
"""
tf.set_random_seed(params['seed'])
np.random.seed(params['seed'])
# Load a trained classifier.
modeldir = os.path.join(params['training_results_dir'], params['clf_name'])
param_file = os.path.join(modeldir, 'params.json')
model_params = utils.load_json(param_file)
if params['preloaded_model'] is None:
ckpt_path = os.path.join(modeldir, 'ckpts/bestmodel-1')
cnn_args = {'conv_dims':
[int(x) for x in model_params['conv_dims'].split(',')],
'conv_sizes':
[int(x) for x in model_params['conv_sizes'].split(',')],
'dense_sizes':
[int(x) for x in model_params['dense_sizes'].split(',')],
'n_classes': model_params['n_classes'], 'onehot': True}
model = utils.load_model(ckpt_path, classifier.CNN, cnn_args)
else:
model = params['preloaded_model']
# Load train/validation/test examples
tensordir = os.path.join(modeldir, 'tensors')
validation_x = utils.load_tensor(os.path.join(tensordir, 'valid_x_infl.npy'))
test_x = utils.load_tensor(os.path.join(tensordir, 'test_x_infl.npy'))
ood_x = utils.load_tensor(os.path.join(tensordir, 'ood_x_infl.npy'))
# Get in- and out-of-distribution classes.
n_labels = model_params['n_classes']
all_classes = range(n_labels)
ood_classes = ([int(x) for x in model_params['ood_classes'].split(',')]
if 'ood_classes' in model_params else [])
ind_classes = [x for x in all_classes if x not in ood_classes]
# Load an iterator of training data.
label_noise = (model_params['label_noise']
if 'label_noise' in model_params else 0.)
# We only look at a portion of the test set for computational reasons.
ninfl = params['n_test_infl']
start_ix = params['start_ix_test_infl']
end_ix = start_ix + ninfl
xinfl_validation = validation_x[start_ix: end_ix]
xinfl_test = test_x[start_ix: end_ix]
xinfl_ood = ood_x[start_ix: end_ix]
# We want to rotate through all the label options.
y_all = tf.concat([tf.one_hot(tf.fill((ninfl,), lab), depth=n_labels)
for lab in ind_classes], axis=0)
y_all = tf.concat([y_all, y_all, y_all], axis=0)
xinfl_validation_all = tf.concat([xinfl_validation for _ in ind_classes],
axis=0)
xinfl_test_all = tf.concat([xinfl_test for _ in ind_classes], axis=0)
xinfl_ood_all = tf.concat([xinfl_ood for _ in ind_classes], axis=0)
x_all = tf.concat([xinfl_validation_all, xinfl_test_all, xinfl_ood_all],
axis=0)
cg_approx_params = {'maxiter': params['cg_maxiter'],
'squared': params['squared'],
'tol': params['tol'],
'hvp_samples': params['hvp_samples']}
# Here we run conjugate gradient one example at a time, collecting
# the following outputs.
# H^{-1}g
infl_value = []
# gH^{-1}g
infl_laplace = []
# H^{-2}g
infl_deriv = []
# g
grads = []
# When calculating H^{-1}g with conjugate gradient, Scipy returns a flag
# denoting the optimization's success.
warning_flags = []
# When calculating H^{-2}g with conjugate gradient, Scipy returns a flag
# denoting the optimization's success.
warning_flags_deriv = []
for i in range(x_all.shape[0]):
logging.info('Example {:d}'.format(i))
s = time.time()
xi = tf.expand_dims(x_all[i], 0)
yi = tf.expand_dims(y_all[i], 0)
if params['preloaded_itr'] is None:
itr_train, _, _, _ = dataset_utils.load_dataset_ood_supervised_onehot(
ind_classes, ood_classes, label_noise=label_noise)
else:
itr_train = params['preloaded_itr']
infl_value_i, grads_i, warning_flag_i = get_parameter_influence(
model, xi, yi, itr_train,
approx_params=cg_approx_params,
damping=params['lam'])
t = time.time()
logging.info('IHVP calculation took {:.3f} seconds'.format(t - s))
infl_laplace_i = tf.multiply(infl_value_i, grads_i)
infl_value_wtshape = tensor_utils.reshape_vector_as(model.weights,
infl_value_i)
loss_function = calculate_influence.make_loss_fn(model, params['lam'])
gradient_function = calculate_influence.make_grad_fn(model)
map_gradient_function = calculate_influence.make_map_grad_fn(model)
s = time.time()
infl_deriv_i, warning_flag_deriv_i = get_ihvp_conjugate_gradient(
infl_value_wtshape, itr_train,
loss_function, gradient_function, map_gradient_function,
approx_params=cg_approx_params)
t = time.time()
logging.info('Second IHVP calculation took {:.3f} seconds'.format(t - s))
infl_value.append(infl_value_i)
infl_laplace.append(infl_laplace_i)
infl_deriv.append(infl_deriv_i)
grads.append(grads_i)
warning_flags.append(tf.expand_dims(warning_flag_i, 0))
warning_flags_deriv.append(tf.expand_dims(warning_flag_deriv_i, 0))
infl_value = tf.concat(infl_value, axis=0)
infl_laplace = tf.concat(infl_laplace, axis=0)
infl_deriv = tf.concat(infl_deriv, axis=0)
grads = tf.concat(grads, axis=0)
warning_flags = tf.concat(warning_flags, axis=0)
warning_flags_deriv = tf.concat(warning_flags_deriv, axis=0)
res = {}
for infl_res, nm in [(infl_value, 'infl'),
(infl_deriv, 'deriv'),
(infl_laplace, 'laplace'),
(grads, 'grads'),
(warning_flags, 'warnflags'),
(warning_flags_deriv, 'warnflags_deriv')]:
res['valid_{}'.format(nm)] = infl_res[:ninfl * len(ind_classes)]
res['test_{}'.format(nm)] = infl_res[
ninfl * len(ind_classes): 2 * ninfl * len(ind_classes)]
res['ood_{}'.format(nm)] = infl_res[2 * ninfl * len(ind_classes):]
# Save the results of these calculations.
if params['output_dir']:
resdir = utils.make_subdir(params['output_dir'], 'influence_results')
else:
resdir = utils.make_subdir(modeldir, 'influence_results')
tensor_name_template = '{}{}-inv_hvp-cg-ix{:d}-ninfl{:d}'+ (
'_squared' if params['squared'] else '')
infl_tensors = [
(tensor_name_template.format(params['tname'], label, start_ix, ninfl),
res[label]) for label in res.keys()]
utils.save_tensors(infl_tensors, resdir)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
params = FLAGS.flag_values_dict()
tf.set_random_seed(params['seed'])
params['results_dir'] = utils.make_subdir(params['training_results_dir'],
params['expname'])
params['figdir'] = utils.make_subdir(params['results_dir'], 'figs')
params['ckptdir'] = utils.make_subdir(params['results_dir'], 'ckpts')
params['logdir'] = utils.make_subdir(params['results_dir'], 'logs')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
# Load the classification model.
conv_dims = [
int(x) for x in (
params['conv_dims'].split(',') if params['conv_dims'] else [])
]
conv_sizes = [
int(x) for x in (
params['conv_sizes'].split(',') if params['conv_sizes'] else [])
]
dense_sizes = [
int(x) for x in (
params['dense_sizes'].split(',') if params['dense_sizes'] else [])
]
params['n_layers'] = len(conv_dims)
clf = classifier.CNN(conv_dims,
conv_sizes,
dense_sizes,
params['n_classes'],
onehot=True)
# Checkpoint the initialized model, in case we want to re-run it from there.
utils.checkpoint_model(clf, params['ckptdir'], 'initmodel')
# Load the "in-distribution" and "out-of-distribution" classes as
# separate splits.
ood_classes = [int(x) for x in params['ood_classes'].split(',')]
# We assume we train on all non-OOD classes.
all_classes = range(params['n_classes'])
ind_classes = [x for x in all_classes if x not in ood_classes]
(itr_train, itr_valid, itr_test,
itr_test_ood) = dataset_utils.load_dset_ood_supervised_onehot(
ind_classes,
ood_classes,
label_noise=(params['label_noise']),
dset_name=params['dataset_name'])
# Train and test the model in-distribution, and save test outputs.
train_cnn.train_classifier(clf, itr_train, itr_valid, params)
train_cnn.test_classifier(clf, itr_test, params, 'test')
# Save model outputs on the training set.
params['tensordir'] = utils.make_subdir(params['results_dir'],
'train_tensors')
train_cnn.test_classifier(clf, itr_train, params, 'train')
# Save model outputs on the OOD set.
params['tensordir'] = utils.make_subdir(params['results_dir'],
'ood_tensors')
train_cnn.test_classifier(clf, itr_test_ood, params, 'ood')
params['tensordir'] = utils.make_subdir(params['results_dir'], 'tensors')
# Save to disk samples of size 1000 from the train, valid, test and OOD sets.
train_data = utils.aggregate_batches(itr_train, 1000,
['train_x_infl', 'train_y_infl'])
validation_data = utils.aggregate_batches(itr_valid, 1000,
['valid_x_infl', 'valid_y_infl'])
test_data = utils.aggregate_batches(itr_test, 1000,
['test_x_infl', 'test_y_infl'])
ood_data = utils.aggregate_batches(itr_test_ood, 1000,
['ood_x_infl', 'ood_y_infl'])
utils.save_tensors(
train_data.items() + validation_data.items() + test_data.items() +
ood_data.items(), params['tensordir'])
def run(params):
"""Calculates influence functions for a pre-loaded model.
params should contain:
seed (int): random seed for Tensorflow and Numpy initialization.
training_results_dir (str): the parent directory of the pre-trained model.
clf_name (str): the name of the pre-trained model's directory.
n_test_infl (int): number of test examples to run influence functions for.
n_train_infl (int): number of train examples to run influence functions for.
lissa_recursion_depth (int): how long to run LiSSA for.
output_dir (str): where results should be written - defaults to
training_results_dir/clf_name/influence_results.
Args:
params (dict): contains a number of params - as loaded from flags.
"""
tf.set_random_seed(params['seed'])
np.random.seed(params['seed'])
# Load a trained classifier.
modeldir = os.path.join(params['training_results_dir'], params['clf_name'])
ckpt_path = os.path.join(modeldir, 'ckpts/bestmodel-1')
param_file = os.path.join(modeldir, 'params.json')
clf_params = utils.load_json(param_file)
cnn_args = {
'conv_dims': [int(x) for x in clf_params['conv_dims'].split(',')],
'conv_sizes': [int(x) for x in clf_params['conv_sizes'].split(',')],
'dense_sizes': [int(x) for x in clf_params['dense_sizes'].split(',')],
'n_classes': 10,
'onehot': True
}
clf = utils.load_model(ckpt_path, classifier.CNN, cnn_args)
# Load train/valid/test examples
tensordir = os.path.join(modeldir, 'tensors')
train_x = utils.load_tensor(os.path.join(tensordir, 'train_x_infl.npy'))
train_y = utils.load_tensor(os.path.join(tensordir, 'train_y_infl.npy'))
test_x = utils.load_tensor(os.path.join(tensordir, 'test_x_infl.npy'))
test_y = utils.load_tensor(os.path.join(tensordir, 'test_y_infl.npy'))
# Calculate influence functions for this model.
if params['output_dir']:
resdir = utils.make_subdir(params['output_dir'], 'influence_results')
else:
resdir = utils.make_subdir(modeldir, 'influence_results')
itr_train, _, _ = dataset_utils.load_dataset_supervised_onehot()
bx, by = test_x[:params['n_test_infl']], test_y[:params['n_test_infl']]
train_loss, _ = clf.get_loss(bx, by)
logging.info('Current loss: {:3f}'.format(tf.reduce_mean(train_loss)))
bx_tr, by_tr = (train_x[:params['n_train_infl']],
train_y[:params['n_train_infl']])
predicted_loss_diffs = get_influence_on_test_loss(
(bx, by), (bx_tr, by_tr),
itr_train,
clf,
approx_params={
'scale': 100,
'damping': 0.1,
'num_samples': 1,
'recursion_depth': params['lissa_recursion_depth'],
'print_iter': 10
},
lam=0.01)
utils.save_tensors([('predicted_loss_diffs', predicted_loss_diffs)],
resdir)
d = predicted_loss_diffs.numpy()
df = d.flatten()
dfsort = sorted(zip(df, range(len(df))),
key=lambda x: abs(x[0]),
reverse=True)
# Ordering note: df[:10] == d[0,:10]
# Loop over the highest influence pairs.
imgs_to_save = []
titles = []
n_img_pairs = 10
for inf_val, idx in dfsort[:n_img_pairs]:
i, tr_i = convert_index(idx, d)
imgs_to_save += [bx[i, :, :, 0], bx_tr[tr_i, :, :, 0]]
titles += [
'Test Image: inf={:.8f}'.format(inf_val),
'Train Image: inf={:.8f}'.format(inf_val)
]
utils.save_images(os.path.join(resdir, 'most_influential.pdf'),
imgs_to_save,
n_img_pairs,
titles=titles)
# For each test image, find the training image with the highest influence
imgs_to_save = []
titles = []
n_img_pairs = 10 # Do this many pairs from the top of the test tensor
for i in range(n_img_pairs):
infl_i = d[i, :]
max_train_ind = np.argmax(infl_i)
max_infl_val = max(infl_i)
imgs_to_save += [bx[i, :, :, 0], bx_tr[max_train_ind, :, :, 0]]
titles += [
'Test Image {:d}: inf={:.8f}'.format(i, max_infl_val),
'Train Image {:d}: inf={:.8f}'.format(max_train_ind, max_infl_val)
]
utils.save_images(os.path.join(resdir, 'most_influential_by_test_img.pdf'),
imgs_to_save,
n_img_pairs,
titles=titles)
