def show_latent_space(model, test_set=False): ''' the option `test_set` controls, whether the test set, or the validation set is used.''' clusters = model.mu.data.cpu().numpy().squeeze() pca = PCA(n_components=2) pca.fit(clusters) mu_red = pca.transform(clusters) z_red = [] true_label = [] data_generator = (data.test_loader if test_set else [(data.val_x, torch.argmax(data.val_y, dim=1))]) with torch.no_grad(): for x, y in data_generator: true_label.append(y.cpu().numpy()) x, y = x.cuda(), data.onehot(y.cuda()) z = model.inn(x).cpu().numpy() z_red.append(pca.transform(z)) z_red = np.concatenate(z_red, axis=0) true_label = np.concatenate(true_label, axis=0) plt.figure() plt.scatter(mu_red[:, 0], mu_red[:, 1], c=np.arange(10), cmap='tab10', s=250, alpha=0.5) plt.scatter(z_red[:, 0], z_red[:, 1], c=true_label, cmap='tab10', s=1) plt.tight_layout()
def calibration_curve(model): pred = [] gt = [] with torch.no_grad(): for x, y in data.test_loader: x, y = x.cuda(), data.onehot(y.cuda()) logits = model(x, y, loss_mean=False)['logits_tr'] #log_pred = logits - torch.logsumexp(logits + np.log(1/10.), dim=1, keepdim=True) #exp_pred = torch.exp(log_pred) #print(torch.mean(torch.max(exp_pred, dim=1)[0])) pred.append(torch.softmax(logits, dim=1).cpu().numpy()) #pred.append(torch.exp(log_pred).cpu().numpy()) gt.append(y.cpu().numpy()) pred = np.concatenate(pred, axis=0).flatten() gt = np.concatenate(gt, axis=0).astype(np.bool).flatten() mask = (pred > 1e-6) mask = mask * (pred < (1 - 1e-6)) pred, gt = pred[mask], gt[mask] n_bins = np.sum(mask) / 50. pred_bins = np.quantile(pred, np.linspace(0., 1., n_bins)) print(np.sum(mask)) correct = pred[gt] wrong = pred[np.logical_not(gt)] hist_correct, _ = np.histogram(correct, bins=pred_bins) hist_wrong, _ = np.histogram(wrong, bins=pred_bins) q = hist_correct / (hist_wrong + hist_correct) p = 0.5 * (pred_bins[1:] + pred_bins[:-1]) poisson_err = q * np.sqrt(1 / hist_correct + 1 / (hist_wrong + hist_correct)) plt.figure(figsize=(10, 10)) plt.errorbar(p, q, yerr=poisson_err, capsize=4, fmt='-o') plt.fill_between(p, q - poisson_err, q + poisson_err, alpha=0.25) plt.plot([0, 1], [0, 1], color='black')
print('Start Iter', iter, ' ...') (X, Y, testX, testY) = data.split_close_buflo('feature/' + folder, 60.0, 40.0, 100.0, mon_instance, dim) # mon_instance, unClass, if exp == 'open': nClass = nClass + 1 if b_label: nClass = 2 # in binary classification, we have only two labels, 0 or -1 print(len(X)) print(len(testX)) print(len(Y)) print(len(testY)) print(nClass) Y = data.onehot(Y, nClass) testY = data.onehot(testY, nClass) X = X.reshape([-1, 1, dim, 1]) testX = testX.reshape([-1, 1, dim, 1]) # Building convolutional network tf.reset_default_graph() network = input_data(shape=[None, 1, dim, 1], name='input') network = conv_2d(network, 32, 3, activation='tanh', regularizer="L2") network = max_pool_2d(network, 2) network = local_response_normalization(network) network = fully_connected(network, 256, activation='tanh') network = dropout(network, 0.8)
mon_instance = 300.0 gb = 'wf' b_label = False dim = 10000 for iter in range(n_iter): print('Start iter', iter, ':::') (X, Y, testX, testY) = data.split_close_defense('feature/' + folder, 60.0, 40.0, 100.0, mon_instance, dim) if exp == 'open': nClass = nClass + 1 if b_label: nClass = 2 ## in binary classification, we have only two labels, 0 or -1 Y = data.onehot(Y, int(nClass)) print(testY) testY = data.onehot(testY, int(nClass)) print(exp, 'nClass:', nClass) # Building deep neural network tf.reset_default_graph() input_layer = tflearn.input_data(shape=[None, dim]) dense1 = tflearn.fully_connected(input_layer, 1345, activation='tanh', weight_decay=0.0) dropout1 = tflearn.dropout(dense1, 0.3) dense2 = tflearn.fully_connected(dropout1, 2315, activation='tanh',
def outlier_histogram(model, test_set=False): ''' the option `test_set` controls, whether the test set, or the validation set is used.''' data_generator = (data.test_loader if test_set else [(data.val_x, torch.argmax(data.val_y, dim=1))]) score, correct_pred = [], [] from torchvision.datasets import FashionMNIST from torch.utils.data import DataLoader fashion_generator = DataLoader(FashionMNIST('./fashion_mnist', download=True, train=False, transform=data.transform), batch_size=data.batch_size, num_workers=8) # continue using dropout for WAIC model.train() def waic(x): waic_samples = 1 ll_joint = [] for i in range(waic_samples): losses = model(x, y=None, loss_mean=False) ll_joint.append(losses['nll_joint_tr'].cpu().numpy()) ll_joint = np.stack(ll_joint, axis=1) return np.mean(ll_joint, axis=1) + np.var(ll_joint, axis=1) with torch.no_grad(): for x, y in data_generator: x, y = x.cuda(), data.onehot(y.cuda()) score.append(waic(x)) losses = model(x, y, loss_mean=False) correct_pred.append( (torch.argmax(y, dim=1) == torch.argmax(losses['logits_tr'], dim=1)).cpu().numpy()) score_fashion = [] for x, y in fashion_generator: x = x.cuda() x = data.augment(x) score_fashion.append(waic(x)) score_adv = [] score_adv_ref = [] adv_images = np.stack( [np.load(f) for f in glob.glob('./adv_examples/adv_*.npy')], axis=0) ref_images = np.stack( [np.load(f) for f in glob.glob('./adv_examples/img_*.npy')], axis=0) adv_images = data.augment(torch.Tensor(adv_images).cuda()) ref_images = data.augment(torch.Tensor(ref_images).cuda()) score_adv = waic(adv_images) score_ref = waic(ref_images) model.eval() score = np.concatenate(score, axis=0) correct_pred = np.concatenate(correct_pred, axis=0) score_fashion = np.concatenate(score_fashion, axis=0) #val_range = [np.quantile(np.concatenate((score, score_fashion, score_adv)), 0.01), #np.quantile(np.concatenate((score, score_fashion, score_adv)), 0.6)] val_range = [-8, 0] val_range[0] -= 0.03 * (val_range[1] - val_range[0]) val_range[1] += 0.03 * (val_range[1] - val_range[0]) bins = 40 plt.figure() plt.hist(score[correct_pred == 1], bins=bins, range=val_range, histtype='step', label='correct', density=True, color='green') plt.hist(score[correct_pred == 0], bins=3 * bins, range=val_range, histtype='step', label='not correct', density=True, color='red') plt.hist(score_fashion, bins=bins, range=val_range, histtype='step', label='$\mathcal{Fashion}$', density=True, color='magenta') plt.hist(score_adv, bins=bins, range=val_range, histtype='step', label='Adv attacks', density=True, color='blue') plt.hist(score_ref, bins=bins, range=val_range, histtype='step', label='Non-attacked images', density=True, color='gray') plt.legend()