def main(args): with torch.cuda.device(1): # ??? Remove this: labels = list(range(10)) batch_size, lr, num_workers, latent_size = int(args.batch_size), float(args.lr), int(args.num_workers), int(args.latent_size) IM_SIZE, model_loc, num_times, iterations = int(args.IM_SIZE), Path(args.model_loc), int(args.num_times), int(args.iterations) device = torch.device(args.device) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) files_df['val'] = files_df['val'].sample(n=int(args.df_sample_num)) if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, label=None, return_loc=True) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, label=None, return_loc=True, normalize=True) # Train for each of the labels, here the model_loc is not an actual loc, just the base if args.model_type == 'vae': model_dict = {} model_file = Path(model_loc).name model_name = model_file.split('-')[0] for label in labels: model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar' model_loc_cr = str(model_loc.parent / model_file_cr) model_dict[label] = load_net(model_loc_cr).to(device).eval() results = eval_gen_vae(dataloaders['val'], model_dict, device, num_times=num_times, iterations=iterations, latent_size=latent_size) elif args.model_type == 'cvae': model = load_net(model_loc).to(device).eval() results = eval_gen_cvae(dataloaders['val'], model, labels, device, num_times=num_times, iterations=iterations, latent_size=latent_size) SAVE_PATH = (str(model_loc).rsplit('-', 1)[:-1][0]+'_iter'+str(iterations)+ '_nt'+str(num_times)+'_nsamp'+str(args.df_sample_num)+'_results.pkl') pickle.dump(results, open(str(SAVE_PATH), "wb"))
def main(args): """Evolve to find the optimal top k-pooling for each layer / block""" print('CUDA VERSION:', torch.version.cuda) batch_size, num_workers, IM_SIZE, epochs = int(args.batch_size), int( args.num_workers), int(args.IM_SIZE), int(args.epochs) device = torch.device(args.device) NPARAMS = 4 # there are 4 blocks for topk NPOPULATION = int(args.NPOPULATION) # population size MAX_GENERATIONS = int(args.MAX_GENERATIONS) # number of generations within_block_act, after_block_act = str(args.within_block_act), str( args.after_block_act) model_name = 'PResNetTopK-' + str(within_block_act) + '_' + str( after_block_act) SAVE_PATH = Path(args.SAVE_PATH) SAVE_PATH.mkdir(parents=True, exist_ok=True) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) group_list = [1, 1, 1, 1] # solutions generated from N(0, 1). later transformed [0, 1] with inv cdf es = cma.CMAEvolutionStrategy(NPARAMS * [0], 1) history = {} history['xbest'] = [] history['fbest'] = [] history['xfavorite'] = [] history['NPOPULATION'] = NPOPULATION history['MAX_GENERATIONS'] = MAX_GENERATIONS for j in tqdm(range(MAX_GENERATIONS)): solutions = es.ask() fitness_list = np.zeros(es.popsize) # ??? Make generators for each generation. Does this matter? or just do it once??? if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False, normalize=True) # evaluate each set of learning rates, using new model each time: for i in range(es.popsize): # convert the normal to a topk probability: topk_list = [norm.cdf(x) for x in solutions[i]] # Create a model with this topk and train it: model = get_PResNetTopK18(within_block_act=within_block_act, after_block_act=after_block_act, frac_list=topk_list, group_list=group_list, num_classes=10) model = model.to(device) metrics = train_net_evol(model, dataloaders, batch_size, epochs, device) # the fitness is the best validation accuracy *-1, because it tries to minimize fitness_list[i] = -1 * metrics['best_val_acc'] es.tell(solutions, fitness_list) # es.logger.add() es.disp() result = es.result history['xbest'].append(result.xbest) history['fbest'].append(result.fbest) history['xfavorite'].append( result.xfavorite) # this is a weird one, maybe try it out print("fitness at generation", (j + 1), result[1]) print("local optimum discovered by solver:\n", result[0]) print("fitness score at this local optimum:", result[1]) print('es.result_pretty-------------------') es.result_pretty() pickle.dump( history, open( str(SAVE_PATH) + '/' + str(model_name) + '_bs_' + str(batch_size) + '_nGen_' + str(MAX_GENERATIONS) + '_nPop_' + str(NPOPULATION) + '_ep_' + str(NPARAMS) + '_history.pkl', "wb"))
def get_adv_perf(args): sample_num, IM_SIZE, latent_size= int(args.sample_num), int(args.IM_SIZE), int(args.latent_size) model_loc = Path(args.model_loc) RESULT_PATH = Path(args.model_loc).parent device = torch.device(args.device) distance = str(args.distance) num_workers = 0 ## ??? what? batch_size = 1 num_classes = 10 labels = list(range(num_classes)) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) files_df['val'] = files_df['val'].sample(n=sample_num) # Make generators: if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=True, normalize=False) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=True, normalize=False) mean = 0.1307 std = 0.3081 transform = transforms.Compose([ transforms.ToPILImage(), transforms.Resize(IM_SIZE), transforms.ToTensor(), transforms.Normalize((mean,), (std,))]) else: print("Incorrect Dataset") # if args.model_type == 'vae': # then the model_loc isn't the real location # model_dict = {} # model_file = Path(model_loc).name # model_name = model_file.split('-')[0] # for label in labels: # model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar' # model_loc_cr = str(model_loc.parent / model_file_cr) # model_dict[label] = load_net(model_loc_cr).to(device).eval() # gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).eval() # elif args.model_type == 'cvae': # model = load_net(model_loc).to(device).eval() # gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).eval() # Train for each of the labels, here the model_loc is not an actual loc, just the base if args.model_type == 'vae' or args.model_type == 'feat_vae': model_dict = {} model_file = Path(model_loc).name model_name = model_file.split('-')[0] for label in labels: model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar' model_loc_cr = str(model_loc.parent / model_file_cr) model_dict[label] = load_net(model_loc_cr, args).to(device).eval() print('args.model_type', args.model_type) if args.model_type == 'vae': print(f'Loading VAE') gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval() elif args.model_type == 'feat_vae': print(f'Loading FEATURE VAE') gen_model = GenerativeFeatVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval() else: print("Invalid model_type") # If CVAE, load model and predict normally: elif args.model_type == 'cvae': print(f'Loading CVAE') model = load_net(model_loc, args).to(device).eval() gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).to(device).eval() else: print("Incorrect Model Type") # ATTACK: fmodel = foolbox.models.PyTorchModel(gen_model, bounds=(0, 1), preprocessing=(mean, std), num_classes=num_classes, device=device) # attack = get_attack(args.attack_type, fmodel, distance=distance) num_failed = 0 results = pd.DataFrame() for i, batch in enumerate(tqdm(dataloaders['val'])): # Foolbox always wants numpy arrays, and we are using single batch, so this batch dim is removed. img, label, file_loc = batch[0].to(device), batch[1].to(device), batch[2][0] # image, label, file_loc = batch[0].numpy(), int(batch[1].numpy()), batch[2][0] # image = load_image(file_loc) See if should use this with the foolbox preprocessign instead # np_img = np.expand_dims(np.squeeze(img.numpy()), 0) # np_img = np.squeeze(img.cpu().numpy()) # trans_img = transform(img.squeeze().unsqueeze(0).cpu()) # trans_img = trans_img.unsqueeze(0).to(device) # trans_orig_pred = np.argmax(gen_model(trans_img).cpu().numpy()) np_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0) np_label = int(label.cpu().numpy()[0]) # print('original prediction foolbox: ', np.argmax(fmodel.predictions(np_img))) # if args.attack_type == 'boundary': # # adv_object = attack(np_img, np_label, unpack=False, tune_batch_size=False, verbose=True, log_every_n_steps=1) # else: # adv_object = attack(np_img, np_label, unpack=False) # $$$$$$$$$$$$$$$$$$ att = fa.PointwiseAttack(fmodel) metric = foolbox.distances.L0 criterion = foolbox.criteria.Misclassification() # Estimate gradients from scores if not gen_model.has_grad: GE = foolbox.gradient_estimators.CoordinateWiseGradientEstimator(0.1) fmodel = foolbox.models.ModelWithEstimatedGradients(fmodel, GE) # gernate Adversarial a = foolbox.adversarial.Adversarial(fmodel, criterion, np_img, np_label, distance=metric) att(a) print('pred', np.argmax(fmodel.predictions(a.image))) print('distance', a.normalized_distance) # $$$$$$$$$$$$$$$$$$ adv_distance = adv_object.distance np_adv_img = adv_object.image # adv_img = torch.from_numpy(np_adv_img).float().unsqueeze(0).to(device) # orig_pred = np.argmax(gen_model(img).cpu().numpy()) # adv_pred = np.argmax(gen_model(adv_img).cpu().numpy()) # np_trans_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0) # trans_adv_img = attack(trans_img, np_label) # trans_adv_pred = np.argmax(gen_model(trans_adv_img).cpu().numpy()) f_orig_pred = np.argmax(fmodel.predictions(np_img)) f_adv_pred = np.argmax(fmodel.predictions(np_adv_img)) # print(f'FOOLBOX: orig_pred: {f_orig_pred}, adv_pred: {f_adv_pred}') # print(f'Original Label {np_label}') if np_adv_img is None: print('-----np_adv_img is None, Attack Failed') num_failed +=1 results = results.append({'path': file_loc, 'true_label': np.squeeze(label.item()), 'adv_distance': adv_distance.value, 'f_orig_pred': f_orig_pred, 'f_adv_pred': f_adv_pred, 'attack_type': attack_type, 'distance': distance}, ignore_index=True) save_loc = str(RESULT_PATH)+'/results_'+str(Path(model_loc).name)+'_ns'+str(sample_num)+'_'+str(attack_type)+'_'+str(distance)+'_adv.pkl' print(f'saving at: {save_loc}') with open(save_loc, 'wb') as f: pickle.dump(results, f, pickle.HIGHEST_PROTOCOL) print(f'Finished testing {sample_num} images. Number of failures: {num_failed}')
def main(args): print('CUDA VERSION:', torch.version.cuda) batch_size, num_workers = int(args.batch_size), int(args.num_workers) IM_SIZE = int(args.IM_SIZE) device = torch.device(args.device) NPARAMS = int(args.epochs) # one learning rate (parameter) per epoch NPOPULATION = int(args.NPOPULATION) # population size MAX_GENERATIONS = int(args.MAX_GENERATIONS) # number of generations SAVE_PATH = Path(args.SAVE_PATH) SAVE_PATH.mkdir(parents=True, exist_ok=True) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) # is std of 2 the right way? or too much?. Normally use +13 std for sol, but because using smaller pop use larger one. es = cma.CMAEvolutionStrategy(NPARAMS * [-2], 2) # solutions generated from N(-2, 1), but transformed to 10^sol history = {} history['xbest'] = [] history['fbest'] = [] history['xfavorite'] = [] history['NPOPULATION'] = NPOPULATION history['MAX_GENERATIONS'] = MAX_GENERATIONS for j in tqdm(range(MAX_GENERATIONS)): solutions = es.ask() fitness_list = np.zeros(es.popsize) # ??? Make generators for each generation. Does this matter? or just do it once??? if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False, normalize=True) # evaluate each set of learning rates, using new model each time: for i in range(es.popsize): model, model_name = net_from_args(args, num_classes=10, IM_SIZE=IM_SIZE) model = model.to(device) # convert the exponenet to a learning rate: lr_list = np.power(10, solutions[i]).tolist() # Train it for the given lr list: metrics = train_net(model, dataloaders, lr_list, batch_size, device) # the fitness is the best validation accuracy *-1, because it tries to minimize fitness_list[i] = -1 * metrics['best_val_acc'] es.tell(solutions, fitness_list) # es.logger.add() es.disp() result = es.result history['xbest'].append(result.xbest) history['fbest'].append(result.fbest) history['xfavorite'].append(result.xfavorite) # this is a weird one, maybe try it out print("fitness at generation", (j+1), result[1]) print("local optimum discovered by solver:\n", result[0]) print("fitness score at this local optimum:", result[1]) print('es.result_pretty-------------------') es.result_pretty() pickle.dump(history, open(str(SAVE_PATH)+'/'+str(model_name)+'_bs_'+str(batch_size)+'_nGen_'+str(MAX_GENERATIONS)+'_nPop_'+str(NPOPULATION)+'_ep_'+str(NPARAMS)+'_history.pkl', "wb"))
def main(args): with torch.cuda.device(1): # ??? Remove this: if args.layer_sizes: args.layer_sizes = [int(i) for i in args.layer_sizes] epochs, batch_size, lr, num_workers = int(args.epochs), int(args.batch_size), float(args.lr), int(args.num_workers) num_labels, IM_SIZE= int(args.num_labels), int(args.IM_SIZE) device = torch.device(args.device) SAVE_PATH = Path(args.SAVE_PATH) SAVE_PATH.mkdir(parents=True, exist_ok=True) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) # Train for each of the labels: for label in range(num_labels): if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, label=label, return_loc=False) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, label=label, return_loc=False) # get the network model, model_name = vae_from_args(args) model = model.to(device) print('next(model.parameters()).device', next(model.parameters()).device) print(f'--------- Training: {model_name} ---------') # get training parameters and train: optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, step_size=int(epochs/4), gamma=0.2) # close enough metrics = {} metrics['train_losses'] = [] metrics['val_losses'] = [] best_val_loss = 100000000 criterion = model.loss for epoch in range(epochs): # train for one epoch train_losses = train_epoch_auto(epoch, lr, dataloaders['train'], model, optimizer, criterion, device) metrics['train_losses'].append(train_losses) # evaluate on validation set val_losses = validate_epoch_auto(dataloaders['val'], model, criterion, device) metrics['val_losses'].append(val_losses) scheduler.step() # remember best validation accuracy and save checkpoint is_best = val_losses < best_val_loss best_val_loss = min(val_losses, best_val_loss) save_checkpoint({ 'model_name': model_name, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), 'epoch': epoch + 1, 'best_val_loss': best_val_loss, 'metrics': metrics, }, is_best, model_name+'_label_'+str(label), SAVE_PATH) RESULT_PATH = str(SAVE_PATH)+'/'+str(model_name)+'_label_'+str(label)+'_metrics.pkl' print('Saving results at:', RESULT_PATH) pickle.dump(metrics, open(RESULT_PATH, "wb"))
def main(args): with torch.cuda.device(1): epochs, batch_size, lr, num_workers = int(args.epochs), int(args.batch_size), float(args.lr), int(args.num_workers) IM_SIZE = int(args.IM_SIZE) device = torch.device(args.device) SAVE_PATH = Path(args.SAVE_PATH) SAVE_PATH.mkdir(parents=True, exist_ok=True) with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) # Make generators: if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=False, normalize=True) # get the network model, model_name = net_from_args(args, num_classes=10, IM_SIZE=IM_SIZE) model = model.to(device) print(f'--------- Training: {model_name} ---------') # get training parameters and train: criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=5e-4) scheduler = lr_scheduler.StepLR(optimizer, step_size=int(epochs/3), gamma=0.2) # close enough metrics = {} metrics['train_top1_acc'] = [] metrics['train_losses'] = [] metrics['val_top1_acc'] = [] metrics['val_losses'] = [] best_val_acc = 0 for epoch in tqdm(range(epochs)): # train for one epoch train_top1_acc, train_losses = train_epoch(dataloaders['train'], model, criterion, optimizer, epoch, device) metrics['train_top1_acc'].append(train_top1_acc) metrics['train_losses'].append(train_losses) # evaluate on validation set val_top1_acc, val_losses = validate_epoch(dataloaders['val'], model, device, criterion=None) metrics['val_top1_acc'].append(val_top1_acc) metrics['val_losses'].append(val_losses) scheduler.step() # remember best validation accuracy and save checkpoint is_best = val_top1_acc > best_val_acc best_val_acc = max(val_top1_acc, best_val_acc) save_checkpoint({ 'model_name': model_name, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), 'epoch': epoch + 1, 'best_val_acc': best_val_acc, 'metrics': metrics, }, is_best, model_name, SAVE_PATH) pickle.dump(metrics, open(str(SAVE_PATH)+'/'+str(model_name)+'_metrics.pkl', "wb"))
def main(args): batch_size, lr, num_workers, latent_size = int(args.batch_size), float( args.lr), int(args.num_workers), int(args.latent_size) IM_SIZE, model_loc, num_times, iterations = int(args.IM_SIZE), Path( args.model_loc), int(args.num_times), int(args.iterations) device = torch.device(args.device) deter = args.deter num_workers = 2 ## do 0 if doesnt work num_classes = 10 labels = list(range(num_classes)) all_results = pd.DataFrame() # Get file sample and make generators: with open(args.files_df_loc, 'rb') as f: files_df = pickle.load(f) files_df['val'] = files_df['val'].sample(n=int(args.df_sample_num)) if args.dataset == 'CIFAR10': dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=True) elif args.dataset == 'MNIST': dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE, path_colname='path', adv_path_colname=None, return_loc=True, normalize=True) # Train for each of the labels, here the model_loc is not an actual loc, just the base if args.model_type == 'vae' or args.model_type == 'feat_vae': print(f'Loading VAE with Deterministic forward pass: {deter}') model_dict = {} model_file = Path(model_loc).name model_name = model_file.split('-')[0] for label in labels: model_file_cr = model_file + '_label_' + str( label) + '_model_best.pth.tar' model_loc_cr = str(model_loc.parent / model_file_cr) model_dict[label] = load_net(model_loc_cr, args).to(device).eval() if args.model_type == 'vae': gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device) elif args.model_type == 'feat_vae': gen_model = GenerativeFeatVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device) else: print("Invalid model_type") # If CVAE, load model and predict normally: elif args.model_type == 'cvae': print(f'Loading CVAE with Deterministic forward pass: {deter}') model = load_net(model_loc, args).to(device).eval() gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).eval() with torch.no_grad(): for i, (tensor_img, label, path) in enumerate(tqdm(dataloaders['val'])): tensor_img, label, file_loc = tensor_img.to(device), label.to( device), path[0] preds, results = gen_model(tensor_img, iterations=iterations, num_times=num_times, info=True, deterministic=deter) pred = np.argmax(preds.cpu().numpy()) img_results = {} # convert to dict so can add to pandas for ind in labels: img_results[ind] = results[ind] img_results['path'] = path img_results['true_label'] = int(label.cpu().numpy()) img_results['path'] = path img_results['predicted_label'] = int(pred) all_results = all_results.append(img_results, ignore_index=True) SAVE_PATH = (str(model_loc).rsplit('-', 1)[:-1][0] + '_iter' + str(iterations) + '_nt' + str(num_times) + '_nsamp' + str(args.df_sample_num)) if deter: SAVE_PATH = SAVE_PATH + '_deter' SAVE_PATH = SAVE_PATH + '_results.pkl' print(f'saving at: {SAVE_PATH}') pickle.dump(all_results, open(str(SAVE_PATH), "wb"))