def train_resnet50(self, epochs=50, n_models=8):
PATH = Path('/home/rene/data/cifar')
save_path = PATH / 'models'
save_path.mkdir(parents=True, exist_ok=True)
num_workers = 6
batch_size = 80
epochs = 40
models = {}
performance = {}
for i in range(8):
models['resnet50_' + str(i)] = resnet50
dataloaders, dataset_sizes = make_batch_gen(str(PATH),
batch_size,
num_workers,
valid_name='valid',
size=197)
for model_name, model_arch in models.items():
model = model_arch(pretrained=False)
model.fc = nn.Linear(model.fc.in_features, 10)
model = model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.005)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer,
step_size=int(epochs / 3),
gamma=0.2)
best_acc, model = train_model(model, criterion, optimizer,
exp_lr_scheduler, epochs,
dataloaders, dataset_sizes)
torch.save(model.state_dict(), str(save_path / model_name))
def evaluate_models(orig_model, bias_model, test_data, runlog):
if orig_model is not None:
orig_model.model.eval()
print('\nEvaluating original and biased models...')
models = {'orig': orig_model, 'bias': bias_model}
else:
print('\nEvaluating biased model...')
models = {'bias': bias_model}
bias_model.model.eval()
for name, model in models.items():
y_true = []
y_pred = []
biased = []
length = len(test_data)
for i in tqdm.tqdm(range(length), total=length):
example = test_data[i]
true = example['label' if name == 'orig' else 'bias_label']
with torch.no_grad():
pred = model.predict(example['image']).item()
y_true.append(true)
y_pred.append(pred)
biased.append(example['biased'])
y_true = np.array(y_true)
y_pred = np.array(y_pred)
biased = np.array(biased)
acc = float(accuracy_score(y_true, y_pred))
f1 = float(f1_score(y_true, y_pred))
acc_r = float(accuracy_score(y_true[biased], y_pred[biased]))
acc_nr = float(accuracy_score(y_true[~biased], y_pred[~biased]))
runlog[name + '_test_acc'] = acc
runlog[name + '_test_f1'] = f1
runlog[name + '_R_acc'] = acc_r
runlog[name + '_NR_acc'] = acc_nr
print('\t{} MODEL TEST ACC: {:.4f}'.format(name.upper(), acc))
print('\t{} MODEL TEST F-1: {:.4f}'.format(name.upper(), f1))
print('\t{} MODEL R Accuracy: {:.4f}'.format(name.upper(), acc_r))
print('\t{} MODEL NR Accuracy: {:.4f}'.format(name.upper(), acc_nr))
print()
if orig_model is not None:
# legacy (for plotting)
runlog['results'] = [
[runlog['orig_R_acc'], runlog['orig_NR_acc']],
[runlog['bias_R_acc'], runlog['bias_NR_acc']],
]
def get_all_perturbations(models, layers, seed=0, batch_size=64):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('Getting perturbations for all models...')
results = {}
for model_name, model in models.items():
print(f'Starting power method for {model_name}...')
mfe = ModelFeatureExtracter(model, layers[model_name])
adv_attack = create_adversarial_attack(mfe, device=device)
fix_seed(seed)
train_dataloader = get_images_dataloader(DATA_PATH, batch_size, transforms=get_images_transforms())
adv_attack.fit(mfe, train_dataloader)
print('Done power method!')
results[model_name] = adv_attack.get_perturbation().cpu()
return results
def evalModels(models, test_loader, testing_mode=False, return_y=False):
test_correct = {key: 0.0 for key in models}
if return_y:
y_pred = {key: torch.Tensor([]).long() for key in models}
y_true = torch.Tensor([]).long()
bar = pyprind.ProgPercent(len(test_loader.dataset),
title="Testing epoch : ")
for model in models.values():
model.train(testing_mode)
with torch.no_grad():
for idx, data in enumerate(test_loader):
x, y = data
inputs = x.to(device)
labels = y.to(device)
if return_y:
y_true = torch.cat((y_true, y.long().view(-1)))
for key, model in models.items():
outputs = model(inputs)
test_correct[key] += (torch.max(
outputs, 1)[1] == labels.long().view(-1)).sum().item()
if return_y:
y_pred[key] = torch.cat(
(y_pred[key],
torch.max(outputs,
1)[1].to(torch.device('cpu')).long()))
bar.update(test_loader.batch_size)
#clear_output(wait=True)
#print('Testing batch : {:.3f} %'.format(
# ((idx+1)*test_loader.batch_size*100) / len(test_loader.dataset)
#))
if return_y:
return test_correct, y_true, y_pred
else:
return test_correct
def run_all_experiments():
device = 'cuda' if torch.cuda.is_available() else 'cpu'
results = defaultdict(dict)
models = {
'vgg16': torchvision.models.vgg16(pretrained=True),
'vgg19': torchvision.models.vgg19(pretrained=True),
'resnet50': torchvision.models.resnet50(pretrained=True)
}
layers = {
'vgg16': models['vgg16'].features[4],
'vgg19': models['vgg19'].features[4],
'resnet50': models['resnet50'].maxpool
}
all_perturbations = get_all_perturbations(models, layers)
raw_imgs_dataloader = get_images_dataloader(DATA_PATH, 128, transforms=get_images_transforms())
for model_name, model in models.items():
mfe = ModelFeatureExtracter(model, layers[model_name])
adv_attack = create_adversarial_attack(mfe, device=device)
print(f'Getting initial predictions for {model_name}...')
initial_predictions = adv_attack.predict_raw(mfe, raw_imgs_dataloader)
for pert_name, perturbation in all_perturbations.items():
pert_dataloader = get_images_dataloader(DATA_PATH, 128, transforms=get_images_transforms(perturbation))
print(f'Getting perturbated predictions for {model_name} with perturbation `{pert_name}`...')
pert_predictions = adv_attack.predict_raw(mfe, pert_dataloader)
fooling_rate = AdversarialAttack.fooling_rate(
initial_predictions['predictions'],
pert_predictions['predictions']
)
print(f'Got {fooling_rate} fooling_rate for perturbation `{pert_name}` when evaluating on {model_name}')
results[model_name][pert_name] = fooling_rate
return results
"ssd": {
"model":
torch.hub.load('NVIDIA/DeepLearningExamples:torchhub',
'nvidia_ssd',
model_math="fp32"),
"path":
"trace"
},
"faster_rcnn": {
"model": models.detection.fasterrcnn_resnet50_fpn(pretrained=True),
"path": "script"
}
}
# Download sample models
for n, m in models.items():
print("Downloading {}".format(n))
m["model"] = m["model"].eval().cuda()
x = torch.ones((1, 3, 300, 300)).cuda()
if m["path"] == "both" or m["path"] == "trace":
trace_model = torch.jit.trace(m["model"], [x])
torch.jit.save(trace_model, n + '_traced.jit.pt')
if m["path"] == "both" or m["path"] == "script":
script_model = torch.jit.script(m["model"])
torch.jit.save(script_model, n + '_scripted.jit.pt')
# Sample Interpolation Model (align_corners=False, for Testing Interpolate Plugin specifically)
class Interpolate(nn.Module):
def __init__(self):
super(Interpolate, self).__init__()
def gen_features(models):
return {
name: gen_feature(model, sparisity)
for name, (model, sparisity) in models.items()
}
def load_model(models, root='./model'):
for k, m in models.items():
__load_model(k, m, root)
def save_model(models, root='./model'):
p = {}
for k, m in models.items():
p[k] = __save_model(k, m, root)
return p
def runModels(name,
train_dataset,
test_dataset,
models,
epoch_size,
batch_size,
learning_rate,
optimizer=optim.SGD,
optimizer_option={
'momentum': 0.9,
'weight_decay': 5e-4
},
criterion=nn.CrossEntropyLoss(),
show=True,
testing_mode=False,
callback=None):
train_loader = DataLoader(train_dataset, batch_size=batch_size)
test_loader = DataLoader(test_dataset, batch_size=batch_size)
accs, pre_epoch = load_running_state(name, models)
if accs is None:
accs = {
**{key + "_train": []
for key in models},
**{key + "_test": []
for key in models}
}
if pre_epoch is None:
pre_epoch = 0
else:
pre_epoch += 1
if show:
showAccuracy(title='Epoch [{:4d}]'.format(pre_epoch), **accs)
optimizers = {
key: optimizer(value.parameters(),
lr=learning_rate,
**optimizer_option)
for key, value in models.items()
}
for epoch in range(pre_epoch, epoch_size):
bar = pyprind.ProgPercent(len(train_dataset),
title="Training epoch {} : ".format(epoch +
1))
train_correct = {key: 0.0 for key in models}
test_correct = {key: 0.0 for key in models}
# training multiple model
for model in models.values():
model.train()
for idx, data in enumerate(train_loader):
x, y = data
inputs = x.to(device)
labels = y.to(device).long().view(-1)
for optimizer in optimizers.values():
optimizer.zero_grad()
for key, model in models.items():
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
cur_correct = (torch.max(outputs, 1)[1] == labels).sum().item()
train_correct[key] += cur_correct
for optimizer in optimizers.values():
optimizer.step()
bar.update(batch_size)
#clear_output(wait=True)
#print('Training batch : {:.3f} %'.format(((idx+1)*batch_size*100) / len(train_dataset)))
# testing multiple model
test_correct = evalModels(models,
test_loader,
testing_mode=testing_mode)
for key, value in train_correct.items():
accs[key + "_train"] += [(value * 100.0) / len(train_dataset)]
for key, value in test_correct.items():
accs[key + "_test"] += [(value * 100.0) / len(test_dataset)]
if show:
clear_output(wait=True)
showAccuracy(title='Epoch [{:4d}]'.format(epoch + 1), **accs)
# epoch end
torch.cuda.empty_cache()
save_running_state(name, models, accs, epoch)
if callback:
callback(models, accs)
return accs
def trainAndTestModels(train_dataset,
test_dataset,
models,
epochs,
batch_size,
learning_rate,
optimizer=optim.SGD,
loss_function=nn.CrossEntropyLoss()):
accuracyOfTrainingPerModels = {}
train_loader = DataLoader(train_dataset, batch_size=batch_size)
test_loader = DataLoader(test_dataset, batch_size=batch_size)
for key, model in models.items():
# Initialize the trainAcc and testAcc arrays as zeros
trainAcc = np.zeros(epochs)
testAcc = np.zeros(epochs)
test_y = []
pred_y = []
for epoch in range(epochs):
# train the models
model.train()
for train_inputs, train_labels in train_loader:
train_inputs = train_inputs.to(device)
train_labels = train_labels.to(device).long().view(-1)
optimizer(model.parameters(),
lr=learning_rate,
momentum=0.9,
weight_decay=5e-4).zero_grad()
outputs = model.forward(train_inputs)
loss = loss_function(outputs, train_labels)
loss.backward()
# Update the parameters
optimizer(model.parameters(),
lr=learning_rate,
momentum=0.9,
weight_decay=5e-4).step()
# Pick the maximum value of each row and return its index
trainAcc[epoch] += (torch.max(
outputs, 1)[1] == train_labels).sum().item()
print("Epochs[%3d/%3d] Loss : %f" % (epoch, epochs, loss))
trainAcc[epoch] = trainAcc[epoch] * 100 / len(train_dataset)
accuracyOfTrainingPerModels.update([(key + '_train', trainAcc)])
# test the models
model.eval()
with torch.no_grad():
for test_inputs, test_labels in test_loader:
test_inputs = test_inputs.to(device)
test_labels = test_labels.to(device).long().view(-1)
outputs = model.forward(test_inputs)
testAcc[epoch] += (torch.max(
outputs, 1)[1] == test_labels).sum().item()
test_y = np.append(
test_y,
test_labels.to(torch.device('cpu')).numpy())
pred_y = np.append(
pred_y,
torch.max(outputs,
1)[1].to(torch.device('cpu')).numpy())
testAcc[epoch] = testAcc[epoch] * 100 / len(test_dataset)
accuracyOfTrainingPerModels.update([(key + '_test', testAcc)])
# Plot confusion matrix at the end of epoch
if epoch == epochs - 1:
plotConfusionMatrix(key, test_y, pred_y)
print(accuracyOfTrainingPerModels)
# Free cache
torch.cuda.empty_cache()
return accuracyOfTrainingPerModels
