def cifar_validate(self, posterior_samples, iteration, log=None): model = self.model model.eval() # To turn off dropout that may be present by default test_inputs = self.test_inputs test_labels = self.test_labels model.posterior_samples = posterior_samples model.posterior_weights = [1 for _ in range(len(posterior_samples))] posterior_outputs = utils.posterior_expectation(model, test_inputs) posterior_loss = F.cross_entropy(posterior_outputs.cpu(), test_labels.cpu()) # posterior_loss = F.nll_loss(torch.log(posterior_outputs.cpu()), test_labels.cpu()) posterior_predictions = Loss.inference_prediction( posterior_outputs) # Checked posterior_accuracy = utils.inference_accuracy(posterior_predictions, test_labels) # Checked print("Classification acc: {:.4f}".format(posterior_accuracy)) print("loss: {:.4f}".format(posterior_loss.data[0])) lib.plot.plot('classification acc', posterior_accuracy) if log == 'fake': # Write to log file self.accuracy_log_file.write('{} {}\n'.format( iteration, posterior_accuracy)) self.accuracy_log_file.flush() elif log == 'real': self.real_acc_log_file.write('{} {}\n'.format( iteration, posterior_accuracy)) self.real_acc_log_file.flush() model.train() # To re-enable dropout
def mnist_validate(self, posterior_samples, iteration, log=None): print("MNIST VALIDATE") model = self.model test_inputs = self.test_inputs test_labels = self.test_labels model.posterior_samples = posterior_samples model.posterior_weights = [1 for _ in range(len(posterior_samples))] posterior_outputs = utils.posterior_expectation(model, test_inputs) # posterior_loss = F.cross_entropy(posterior_outputs.cpu(), test_labels) posterior_loss = F.nll_loss(torch.log(posterior_outputs.cpu()), test_labels.cpu()) utils.check_nan(posterior_loss, check_big=True, message="") posterior_predictions = Loss.inference_prediction( posterior_outputs) # Checked posterior_accuracy = utils.inference_accuracy(posterior_predictions, test_labels) # Checked print("Classification acc: {:.4f}".format(posterior_accuracy)) print("loss: {:.4f}".format(posterior_loss.data[0])) lib.plot.plot('classification acc', posterior_accuracy) if log == 'fake': # Write to log file self.accuracy_log_file.write('{} {}\n'.format( iteration, posterior_accuracy)) self.accuracy_log_file.flush() elif log == 'real': self.real_acc_log_file.write('{} {}\n'.format( iteration, posterior_accuracy)) self.real_acc_log_file.flush()
def acq_bald(model, poolset, poolloader, acq_size, arguments, opt_config): # Deteriministic if arguments['--deter']: return acq_random(model, poolset, poolloader, acq_size, arguments) balds = [] model.eval() for _, (inputs, _) in enumerate(poolloader): if arguments['--cuda']: inputs = inputs.cuda() inputs = Variable(inputs, volatile=True) sm, sample_sms = utils.posterior_expectation(model, inputs, keep_samples=True, sample_size=opt_config['test_sample_batch_size']) balds += list(utils.f_bald([sm, sample_sms])) model.train() return np.argsort(balds)[::-1][:acq_size]
def acq_entropy(model, poolset, poolloader, acq_size, arguments, opt_config): Hs = [] model.eval() for _, (inputs, _) in enumerate(poolloader): if arguments['--cuda']: inputs = inputs.cuda() inputs = Variable(inputs, volatile=True) # Deteriministic if arguments['--deter']: outputs = model.forward(inputs) sm = F.softmax(outputs) elif arguments['--mc_dropout_passes']: sm = utils.mc_dropout_expectation(model, inputs, keep_samples=False, passes=opt_config['test_sample_batch_size']) # Bayesian Prediction else: sm = utils.posterior_expectation(model, inputs, sample_size=opt_config['test_sample_batch_size']) Hs += list(utils.f_entropy(sm.data.cpu().numpy())) model.train() return np.argsort(Hs)[::-1][:acq_size]
def main(arguments): ## active learning configuration # fixed (i.e. from Gal'17) initial_data_per_class = 2 acq_size = 10 validation_set_size = 100 # can tune num_acq = 50 iterations_per_acq = 3000 acq_crit = 'acq_' + arguments['<acq_crit>'] name_dataset = arguments['<dataset>'] # Load arguments: Module, Optimizer, Loss_function model, optimizer, Loss, exp_name, model_config, opt_config = load_configuration(arguments, name_dataset) num_iterations = iterations_per_acq if arguments['--apd_gan']: #### #### GAN CONFIG #### gan_config = gan_utils.opt_load_configuration(arguments['--apd_gan'], None) ### compute OUTPUT_DIM posterior_sampling(1, model, 1) gan_config['output_dim'] = _flatten_npyfy(model.posterior_samples).shape[1] model.posterior_samples = [] model.posterior_weights = [] ### obj_traingan = TrainGAN(None, gan_config, model, []) obj_traingan.init_gan() ### gan_bs = gan_config['batch_size'] # Batch size gan_inp_buffer = [] gan_iter = 0 default_config = { 'batcher_kwargs':{'batch_size': 100}, 'burnin_iters': 500, 'sample_size': 10, 'sample_interval': 20, 'validation_interval': 2000, 'ood_scale': 5, 'variance_monitor_interval': 50, 'ood_datasets': ['notMNIST'] } # Fill in default configuration for keys that are not overwritten by the config file for key in default_config: if key not in opt_config: opt_config[key] = default_config[key] # Fill in default n_anom value, if not overwritten by the config file if 'n_anom' not in opt_config: if name_task.split('-')[0] in ['mnist', 'fashion']: opt_config['n_anom'] = 2000 elif name_task.split('-')[0] == 'babymnist': opt_config['n_anom'] = 300 exp_name = '{}{}{}-{}'.format('D' if arguments['--deter'] else 'B', 'W' if arguments['--warm'] else 'C', arguments['<acq_crit>'], exp_name) # Set up batch_size = 100 # alpha_threshold = .5 burnin_iters = opt_config['burnin_iters'] sample_size = 1 if arguments['--deter'] else opt_config['sample_size'] sample_interval = opt_config['sample_interval'] validation_interval = 500 variance_monitor_interval = 50 is_collecting = False pool_batch_size = 100 print('Experiment name: {}'.format(exp_name)) log_folder = os.path.join('./logs', exp_name) if not os.path.exists(log_folder): os.makedirs(log_folder) # Load DataSet # trainLoader automatically generate training_batch transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) if name_dataset == 'babymnist': trainset = BabyMnist(train=True, transform=transform) poolset = BabyMnist(train=True, transform=transform) testset = BabyMnist(train=False, transform=transform) if name_dataset == 'mnist': trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform) poolset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform) testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform) # trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2) testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2) # trainloader, valloader, testloader, name_dataset = utils.get_dataloader(name_dataset, batch_size) # Tensorboard # # Set tensorboard_monitor monitors # loss_monitor = Monitor('cross_entropy') # accuracy_monitor = Monitor('accuracy') # learning_rate_monitor = Monitor('learning_rate') # variance_monitor = Monitor('variance') # Initial tensorboard # sess, train_writer, point_writer, posterior_writer, mcdropout_writer = initial_tensorboard(exp_name) acq_test_acc = [] acq_anom_detection = [] ood_data = utils.load_ood_data(name_dataset, opt_config) ################ ### Training ### ################ for acq_idx in range(num_acq): print("Acquisition Iter {}/{}".format(acq_idx+1, num_acq)) if acq_idx == 0: ### initial setup ### create a class-balanced initial training set init_trainset_idxs = [[] for i in range(10)] counter = 0 for (idx, label) in enumerate(poolset.train_labels): if len(init_trainset_idxs[label]) < initial_data_per_class: init_trainset_idxs[label].append(idx) counter += 1 if counter == initial_data_per_class * 10: break trainset_idxs = np.array(init_trainset_idxs).ravel() cat = False else: ### acquire! if arguments['--apd_gan']: print('COMPRESSING GAN') gan_inp_buffer = _flatten_npyfy(model.posterior_samples) for _ in range(1000): idxs = np.arange(len(gan_inp_buffer)) np.random.shuffle(idxs) obj_traingan.update_iter(gan_iter, np.array(gan_inp_buffer)[idxs[:gan_bs]]) print('DE-COMPRESSING GAN') ## Model gmodel = eval(model_config['name'])(**model_config['kwargs']) gmodel = utils.cuda(gmodel, arguments) ## get the samples gmodel.posterior_samples = obj_traingan.get_samples(len(model.posterior_samples)) gmodel.posterior_weights = [1 for _ in range(len(gmodel.posterior_samples))] ## replace model = gmodel # acquire! trainset_idxs = acq_dict[acq_crit](model, poolset, poolloader, acq_size, arguments, opt_config) cat = True update_train_pool_sets(trainset, poolset, trainset_idxs, cat) ## create loaders print("Current Training Set Size: {}".format(len(trainset.train_labels))) trainloader = torch.utils.data.DataLoader(trainset, batch_size=len(trainset.train_labels), shuffle=True, num_workers=2) poolloader = torch.utils.data.DataLoader(poolset, batch_size=pool_batch_size, shuffle=False, num_workers=2) ## cold/warm-start the model if acq_idx > 0: if not arguments['--warm']: model = eval(model_config['name'])(**model_config['kwargs']) if arguments['--cuda']: model.type(torch.cuda.FloatTensor) # Optimizer optimizer = eval(opt_config['name'])(model.parameters(), **opt_config['kwargs']) else: num_iterations = int(arguments['<fine_tune_iter>']) # preproc train data data = iter(trainloader).next() inputs, labels = data if arguments['--cuda']: inputs = inputs.type(torch.cuda.FloatTensor) labels = labels.cuda() inputs, labels = Variable(inputs), Variable(labels) ## adjust optimizer if opt_config['name'] == 'NoisedSGD': optimizer.correction = float(len(trainset.train_labels)) *1000./2.#... for iteration in range(num_iterations): # Update learning rate learning_rate = update_LearningRate(optimizer, iteration, opt_config) # learning_rate_monitor.record_tensorboard(learning_rate, iteration, sess, train_writer) # Inference optimizer.zero_grad() outputs = model.forward(inputs) training_loss = F.cross_entropy(outputs, labels) training_loss.backward() # Computes gradients of the model parameters wrt the loss prob_outputs = F.softmax(outputs) training_predictions = prob_outputs.data.cpu().numpy().argmax(1) accuracy = utils.inference_accuracy(training_predictions, labels) # accuracy_monitor.record_tensorboard(accuracy, iteration, sess, train_writer) optimizer.step() # monitor Variance if opt_config['name'] == 'NoisedSGD' and is_collecting == False and iteration % variance_monitor_interval == 0: # is_collecting = is_into_langevin_dynamics(model, outputs, gradient_data, optimizer, learning_rate,alpha_threshold,iteration,sess, train_writer) is_collecting = iteration >= burnin_iters if is_collecting and iteration % sample_interval == 0: posterior_sampling(sample_size, model, learning_rate) # Validation if (iteration % validation_interval == 0): point_accuracy = [] point_loss = [] posterior_accuracy = [] posterior_loss = [] posterior_flag = 0 test_inputs_anomoly_detection = [] for i, data in enumerate(testloader, 0): # Load data # data for inference test_inputs, test_labels = data if arguments['--cuda']: test_inputs = test_inputs.type(torch.cuda.FloatTensor) test_labels = test_labels.cuda() test_inputs, test_labels = Variable(test_inputs, volatile=True), Variable(test_labels, volatile=True) model.eval() # Prediction # Point Prediction point_outputs = model.forward(test_inputs) point_loss_batch = F.cross_entropy(point_outputs, test_labels) point_loss.append(point_loss_batch.data[0]) point_predictions = Loss.inference_prediction(point_outputs) point_accuracy_batch = utils.inference_accuracy(point_predictions, test_labels) point_accuracy.append(point_accuracy_batch) # Bayesian Prediction if len(model.posterior_samples) > 0 and opt_config['name'] == 'NoisedSGD': posterior_flag = 1 posterior_outputs = utils.posterior_expectation(model, test_inputs, sample_size=opt_config['test_sample_batch_size']) posterior_loss_batch = F.nll_loss(torch.log(posterior_outputs), test_labels.cpu()) posterior_loss.append(posterior_loss_batch.data[0]) posterior_predictions = Loss.inference_prediction(posterior_outputs) posterior_accuracy_batch = utils.inference_accuracy(posterior_predictions, test_labels) posterior_accuracy.append(posterior_accuracy_batch) # record prediction result point_accuracy = np.mean(point_accuracy) point_loss = np.mean(point_loss) # accuracy_monitor.record_tensorboard(point_accuracy, iteration, sess, point_writer) # loss_monitor.record_tensorboard(point_loss, iteration, sess, point_writer) if posterior_flag == 1: posterior_accuracy = np.mean(posterior_accuracy) posterior_loss = np.mean(posterior_loss) # accuracy_monitor.record_tensorboard(posterior_accuracy, iteration, sess, posterior_writer) # loss_monitor.record_tensorboard(posterior_loss, iteration, sess, posterior_writer) # acq_test_acc.append(posterior_accuracy) # else: # acq_test_acc.append(point_accuracy) # Is this right? # pdb.set_trace() print('It: {} | Trn Loss: {}| Trn acc: {} | Tst point acc: {} | Tst posterior acc: {}'.format(iteration, training_loss.data[0],accuracy, point_accuracy, posterior_accuracy)) # Termination if iteration == num_iterations - 1: # loss_monitor.save_result_numpy(log_folder) print("This iteration of Active Learning finished {}/{}".format(acq_idx+1, num_acq)) print(accuracy, point_accuracy, posterior_accuracy) acq_test_acc.append(posterior_accuracy) # pdb.set_trace() # acq_anom_detection.append((auroc, n_aupr, ab_aupr)) idx = iteration # check_point(model, optimizer, iteration, exp_name) np.save(os.path.join(log_folder, 'acq_acc'), np.array(acq_test_acc)) np.save(os.path.join(log_folder, 'acq_anom'), np.array(acq_anom_detection)) pickle.dump(arguments, open(os.path.join(log_folder, 'args.pkl'), 'wb'))
def evaluate(model, testloader, posterior_samples, posterior_weights, posterior_flag, Loss, opt_config, arguments): model.eval() point_accuracy = [] point_loss = [] posterior_accuracy = [] posterior_loss = [] for i, data in enumerate(testloader, 0): # Load data # data for inference test_inputs, test_labels = data if arguments['--cuda']: test_inputs = test_inputs.cuda() test_labels = test_labels.cuda() test_inputs, test_labels = Variable( test_inputs, volatile=True), Variable(test_labels, volatile=True) # Prediction # Point Prediction point_outputs = model.forward(test_inputs) point_loss_batch = F.cross_entropy(point_outputs.cpu(), test_labels.cpu()) point_loss.append(point_loss_batch.data[0]) # point_predictions = Loss.inference_prediction(point_outputs) prob_inputs = F.softmax(point_outputs) point_predictions = prob_inputs.data.cpu().numpy().argmax(1) point_accuracy_batch = utils.inference_accuracy( point_predictions, test_labels) point_accuracy.append(point_accuracy_batch) # Bayesian Prediction if posterior_flag: posterior_outputs = utils.posterior_expectation(model, test_inputs) # posterior_loss_batch = Loss.nll(torch.log(posterior_outputs), test_labels) posterior_loss_batch = F.nll_loss( torch.log(posterior_outputs.cpu()), test_labels.cpu()) posterior_loss.append(posterior_loss_batch.data[0]) # posterior_predictions = Loss.inference_prediction(posterior_outputs) prob_inputs = F.softmax(posterior_outputs) posterior_predictions = prob_inputs.data.cpu().numpy().argmax(1) posterior_accuracy_batch = utils.inference_accuracy( posterior_predictions, test_labels) posterior_accuracy.append(posterior_accuracy_batch) model.train() # record prediction result point_accuracy = np.mean(point_accuracy) point_loss = np.mean(point_loss) if posterior_flag == 1: posterior_accuracy = np.mean(posterior_accuracy) posterior_loss = np.mean(posterior_loss) return point_accuracy, point_loss, posterior_accuracy, posterior_loss