def train_resnet50(self, epochs=50, n_models=8): PATH = Path('/home/rene/data/cifar') save_path = PATH / 'models' save_path.mkdir(parents=True, exist_ok=True) num_workers = 6 batch_size = 80 epochs = 40 models = {} performance = {} for i in range(8): models['resnet50_' + str(i)] = resnet50 dataloaders, dataset_sizes = make_batch_gen(str(PATH), batch_size, num_workers, valid_name='valid', size=197) for model_name, model_arch in models.items(): model = model_arch(pretrained=False) model.fc = nn.Linear(model.fc.in_features, 10) model = model.cuda() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.005) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=int(epochs / 3), gamma=0.2) best_acc, model = train_model(model, criterion, optimizer, exp_lr_scheduler, epochs, dataloaders, dataset_sizes) torch.save(model.state_dict(), str(save_path / model_name))
def evaluate_models(orig_model, bias_model, test_data, runlog): if orig_model is not None: orig_model.model.eval() print('\nEvaluating original and biased models...') models = {'orig': orig_model, 'bias': bias_model} else: print('\nEvaluating biased model...') models = {'bias': bias_model} bias_model.model.eval() for name, model in models.items(): y_true = [] y_pred = [] biased = [] length = len(test_data) for i in tqdm.tqdm(range(length), total=length): example = test_data[i] true = example['label' if name == 'orig' else 'bias_label'] with torch.no_grad(): pred = model.predict(example['image']).item() y_true.append(true) y_pred.append(pred) biased.append(example['biased']) y_true = np.array(y_true) y_pred = np.array(y_pred) biased = np.array(biased) acc = float(accuracy_score(y_true, y_pred)) f1 = float(f1_score(y_true, y_pred)) acc_r = float(accuracy_score(y_true[biased], y_pred[biased])) acc_nr = float(accuracy_score(y_true[~biased], y_pred[~biased])) runlog[name + '_test_acc'] = acc runlog[name + '_test_f1'] = f1 runlog[name + '_R_acc'] = acc_r runlog[name + '_NR_acc'] = acc_nr print('\t{} MODEL TEST ACC: {:.4f}'.format(name.upper(), acc)) print('\t{} MODEL TEST F-1: {:.4f}'.format(name.upper(), f1)) print('\t{} MODEL R Accuracy: {:.4f}'.format(name.upper(), acc_r)) print('\t{} MODEL NR Accuracy: {:.4f}'.format(name.upper(), acc_nr)) print() if orig_model is not None: # legacy (for plotting) runlog['results'] = [ [runlog['orig_R_acc'], runlog['orig_NR_acc']], [runlog['bias_R_acc'], runlog['bias_NR_acc']], ]
def get_all_perturbations(models, layers, seed=0, batch_size=64): device = 'cuda' if torch.cuda.is_available() else 'cpu' print('Getting perturbations for all models...') results = {} for model_name, model in models.items(): print(f'Starting power method for {model_name}...') mfe = ModelFeatureExtracter(model, layers[model_name]) adv_attack = create_adversarial_attack(mfe, device=device) fix_seed(seed) train_dataloader = get_images_dataloader(DATA_PATH, batch_size, transforms=get_images_transforms()) adv_attack.fit(mfe, train_dataloader) print('Done power method!') results[model_name] = adv_attack.get_perturbation().cpu() return results
def evalModels(models, test_loader, testing_mode=False, return_y=False): test_correct = {key: 0.0 for key in models} if return_y: y_pred = {key: torch.Tensor([]).long() for key in models} y_true = torch.Tensor([]).long() bar = pyprind.ProgPercent(len(test_loader.dataset), title="Testing epoch : ") for model in models.values(): model.train(testing_mode) with torch.no_grad(): for idx, data in enumerate(test_loader): x, y = data inputs = x.to(device) labels = y.to(device) if return_y: y_true = torch.cat((y_true, y.long().view(-1))) for key, model in models.items(): outputs = model(inputs) test_correct[key] += (torch.max( outputs, 1)[1] == labels.long().view(-1)).sum().item() if return_y: y_pred[key] = torch.cat( (y_pred[key], torch.max(outputs, 1)[1].to(torch.device('cpu')).long())) bar.update(test_loader.batch_size) #clear_output(wait=True) #print('Testing batch : {:.3f} %'.format( # ((idx+1)*test_loader.batch_size*100) / len(test_loader.dataset) #)) if return_y: return test_correct, y_true, y_pred else: return test_correct
def run_all_experiments(): device = 'cuda' if torch.cuda.is_available() else 'cpu' results = defaultdict(dict) models = { 'vgg16': torchvision.models.vgg16(pretrained=True), 'vgg19': torchvision.models.vgg19(pretrained=True), 'resnet50': torchvision.models.resnet50(pretrained=True) } layers = { 'vgg16': models['vgg16'].features[4], 'vgg19': models['vgg19'].features[4], 'resnet50': models['resnet50'].maxpool } all_perturbations = get_all_perturbations(models, layers) raw_imgs_dataloader = get_images_dataloader(DATA_PATH, 128, transforms=get_images_transforms()) for model_name, model in models.items(): mfe = ModelFeatureExtracter(model, layers[model_name]) adv_attack = create_adversarial_attack(mfe, device=device) print(f'Getting initial predictions for {model_name}...') initial_predictions = adv_attack.predict_raw(mfe, raw_imgs_dataloader) for pert_name, perturbation in all_perturbations.items(): pert_dataloader = get_images_dataloader(DATA_PATH, 128, transforms=get_images_transforms(perturbation)) print(f'Getting perturbated predictions for {model_name} with perturbation `{pert_name}`...') pert_predictions = adv_attack.predict_raw(mfe, pert_dataloader) fooling_rate = AdversarialAttack.fooling_rate( initial_predictions['predictions'], pert_predictions['predictions'] ) print(f'Got {fooling_rate} fooling_rate for perturbation `{pert_name}` when evaluating on {model_name}') results[model_name][pert_name] = fooling_rate return results
"ssd": { "model": torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_ssd', model_math="fp32"), "path": "trace" }, "faster_rcnn": { "model": models.detection.fasterrcnn_resnet50_fpn(pretrained=True), "path": "script" } } # Download sample models for n, m in models.items(): print("Downloading {}".format(n)) m["model"] = m["model"].eval().cuda() x = torch.ones((1, 3, 300, 300)).cuda() if m["path"] == "both" or m["path"] == "trace": trace_model = torch.jit.trace(m["model"], [x]) torch.jit.save(trace_model, n + '_traced.jit.pt') if m["path"] == "both" or m["path"] == "script": script_model = torch.jit.script(m["model"]) torch.jit.save(script_model, n + '_scripted.jit.pt') # Sample Interpolation Model (align_corners=False, for Testing Interpolate Plugin specifically) class Interpolate(nn.Module): def __init__(self): super(Interpolate, self).__init__()
def gen_features(models): return { name: gen_feature(model, sparisity) for name, (model, sparisity) in models.items() }
def load_model(models, root='./model'): for k, m in models.items(): __load_model(k, m, root)
def save_model(models, root='./model'): p = {} for k, m in models.items(): p[k] = __save_model(k, m, root) return p
def runModels(name, train_dataset, test_dataset, models, epoch_size, batch_size, learning_rate, optimizer=optim.SGD, optimizer_option={ 'momentum': 0.9, 'weight_decay': 5e-4 }, criterion=nn.CrossEntropyLoss(), show=True, testing_mode=False, callback=None): train_loader = DataLoader(train_dataset, batch_size=batch_size) test_loader = DataLoader(test_dataset, batch_size=batch_size) accs, pre_epoch = load_running_state(name, models) if accs is None: accs = { **{key + "_train": [] for key in models}, **{key + "_test": [] for key in models} } if pre_epoch is None: pre_epoch = 0 else: pre_epoch += 1 if show: showAccuracy(title='Epoch [{:4d}]'.format(pre_epoch), **accs) optimizers = { key: optimizer(value.parameters(), lr=learning_rate, **optimizer_option) for key, value in models.items() } for epoch in range(pre_epoch, epoch_size): bar = pyprind.ProgPercent(len(train_dataset), title="Training epoch {} : ".format(epoch + 1)) train_correct = {key: 0.0 for key in models} test_correct = {key: 0.0 for key in models} # training multiple model for model in models.values(): model.train() for idx, data in enumerate(train_loader): x, y = data inputs = x.to(device) labels = y.to(device).long().view(-1) for optimizer in optimizers.values(): optimizer.zero_grad() for key, model in models.items(): outputs = model(inputs) loss = criterion(outputs, labels) loss.backward() cur_correct = (torch.max(outputs, 1)[1] == labels).sum().item() train_correct[key] += cur_correct for optimizer in optimizers.values(): optimizer.step() bar.update(batch_size) #clear_output(wait=True) #print('Training batch : {:.3f} %'.format(((idx+1)*batch_size*100) / len(train_dataset))) # testing multiple model test_correct = evalModels(models, test_loader, testing_mode=testing_mode) for key, value in train_correct.items(): accs[key + "_train"] += [(value * 100.0) / len(train_dataset)] for key, value in test_correct.items(): accs[key + "_test"] += [(value * 100.0) / len(test_dataset)] if show: clear_output(wait=True) showAccuracy(title='Epoch [{:4d}]'.format(epoch + 1), **accs) # epoch end torch.cuda.empty_cache() save_running_state(name, models, accs, epoch) if callback: callback(models, accs) return accs
def trainAndTestModels(train_dataset, test_dataset, models, epochs, batch_size, learning_rate, optimizer=optim.SGD, loss_function=nn.CrossEntropyLoss()): accuracyOfTrainingPerModels = {} train_loader = DataLoader(train_dataset, batch_size=batch_size) test_loader = DataLoader(test_dataset, batch_size=batch_size) for key, model in models.items(): # Initialize the trainAcc and testAcc arrays as zeros trainAcc = np.zeros(epochs) testAcc = np.zeros(epochs) test_y = [] pred_y = [] for epoch in range(epochs): # train the models model.train() for train_inputs, train_labels in train_loader: train_inputs = train_inputs.to(device) train_labels = train_labels.to(device).long().view(-1) optimizer(model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=5e-4).zero_grad() outputs = model.forward(train_inputs) loss = loss_function(outputs, train_labels) loss.backward() # Update the parameters optimizer(model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=5e-4).step() # Pick the maximum value of each row and return its index trainAcc[epoch] += (torch.max( outputs, 1)[1] == train_labels).sum().item() print("Epochs[%3d/%3d] Loss : %f" % (epoch, epochs, loss)) trainAcc[epoch] = trainAcc[epoch] * 100 / len(train_dataset) accuracyOfTrainingPerModels.update([(key + '_train', trainAcc)]) # test the models model.eval() with torch.no_grad(): for test_inputs, test_labels in test_loader: test_inputs = test_inputs.to(device) test_labels = test_labels.to(device).long().view(-1) outputs = model.forward(test_inputs) testAcc[epoch] += (torch.max( outputs, 1)[1] == test_labels).sum().item() test_y = np.append( test_y, test_labels.to(torch.device('cpu')).numpy()) pred_y = np.append( pred_y, torch.max(outputs, 1)[1].to(torch.device('cpu')).numpy()) testAcc[epoch] = testAcc[epoch] * 100 / len(test_dataset) accuracyOfTrainingPerModels.update([(key + '_test', testAcc)]) # Plot confusion matrix at the end of epoch if epoch == epochs - 1: plotConfusionMatrix(key, test_y, pred_y) print(accuracyOfTrainingPerModels) # Free cache torch.cuda.empty_cache() return accuracyOfTrainingPerModels