def exp(net, model_name, attack, test_dataset, device):
original_net = None
image_size = (128, 128)
if model_name == 'baseline':
original_net = BaseCNN()
elif model_name == 'nvidia':
original_net = Nvidia()
elif model_name == 'vgg16':
original_net = Vgg16()
original_net.load_state_dict(torch.load(model_name + '.pt'))
original_net = original_net.to(device)
original_net.eval()
# print(ast_ori, ast_dist)
test_y = pd.read_csv('ch2_final_eval.csv')['steering_angle'].values
test_composed = transforms.Compose(
[Rescale(image_size),
Preprocess('baseline'),
ToTensor()])
test_dataset = UdacityDataset(dataset_path, ['testing'], test_composed,
'test')
test_generator = DataLoader(test_dataset, batch_size=64, shuffle=False)
target = 0.3
ast_ori, _ = fgsm_ex(test_generator, original_net, model_name, target,
device, len(test_dataset))
ast_dist, _ = fgsm_ex(test_generator, net, model_name, target, device,
len(test_dataset))
print('fgsm:', ast_ori, ast_dist)
advt_model = model_name + '_' + attack
ast_ori, _ = advGAN_ex(test_generator, original_net, model_name, target,
device, len(test_dataset))
ast_dist, _ = advGAN_ex(test_generator, net, advt_model, target, device,
len(test_dataset))
print('advGAN:', ast_ori, ast_dist)
advt_model = model_name + '_' + attack
ast_ori, _ = advGAN_uni_ex(test_generator, original_net, model_name,
target, device, len(test_dataset))
ast_dist, _ = advGAN_uni_ex(test_generator, net, advt_model, target,
device, len(test_dataset))
print('advGAN_uni:', ast_ori, ast_dist)
advt_model = model_name + '_' + attack
ast_ori, _ = opt_uni_ex(test_generator, original_net, model_name, target,
device, len(test_dataset))
ast_dist, _ = opt_uni_ex(test_generator, net, advt_model, target, device,
len(test_dataset))
print('opt_uni:', ast_ori, ast_dist)
ast_ori, _ = opt_ex(test_dataset, original_net, model_name, target, device,
len(test_dataset))
ast_dist, _ = opt_ex(test_dataset, net, model_name, target, device,
len(test_dataset))
print('opt:', ast_ori, ast_dist)
def advGAN_Attack(model_name, target_model_path, target, train_dataset, universal=False):
image_nc=3
epochs = 60
batch_size = 64
BOX_MIN = 0
BOX_MAX = 1
# target = 0.2
# Define what device we are using
print("CUDA Available: ",torch.cuda.is_available())
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
image_size = (128, 128)
if 'baseline' in target_model_path:
targeted_model = BaseCNN().to(device)
#image_size = (128, 128)
elif 'nvidia' in target_model_path:
targeted_model = Nvidia().to(device)
#image_size = (66, 200)
elif 'vgg16' in target_model_path:
targeted_model = Vgg16().to(device)
#image_size = (224, 224)
targeted_model.load_state_dict(torch.load(target_model_path))
targeted_model.eval()
if not universal:
advGAN = AdvGAN_Attack(
device,
targeted_model,
model_name,
target,
image_nc,
BOX_MIN,
BOX_MAX)
else:
advGAN = AdvGAN_Uni_Attack(
device,
targeted_model,
model_name,
image_size,
target,
image_nc,
BOX_MIN,
BOX_MAX)
advGAN.train(train_dataset, epochs)
return advGAN
def cal_detection_rate():
for model_name in ['baseline', 'nvidia', 'vgg16']:
# model_name = 'vgg16'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if 'baseline' in model_name:
net = BaseCNN()
elif 'nvidia' in model_name:
net = Nvidia()
elif 'vgg16' in model_name:
net = Vgg16(False)
net.load_state_dict(torch.load(model_name + '.pt'))
net.eval()
net = net.to(device)
dataset_path = '../udacity-data'
root_dir = dataset_path
test_composed = transforms.Compose([Rescale((128, 128)), Preprocess('baseline'), ToTensor()])
image_size = (128, 128)
full_dataset = UdacityDataset(root_dir, ['testing'], test_composed, type_='test')
full_indices = list(range(5614))
test_indices = list(np.random.choice(5614, int(0.2*5614), replace=False))
train_indices = list(set(full_indices).difference(set(test_indices)))
train_dataset = torch.utils.data.Subset(full_dataset, train_indices)
test_dataset = torch.utils.data.Subset(full_dataset, test_indices)
test_data_loader = DataLoader(full_dataset, batch_size=1, shuffle=False)
num_sample = len(full_dataset)
target = 0.3
# attack_detection(model_name, net, test_data_loader, attack='fgsm')
# attack_detection(model_name, net, test_data_loader, attack='advGAN')
# attack_detection(model_name, net, test_data_loader, attack='advGAN_uni')
# attack_detection(model_name, net, test_data_loader, attack='opt_uni')
# attack_detection(model_name, net, test_data_loader, attack='opt')
print('threshold', 0.01)
attack_detection(model_name, net, test_data_loader, attack='fgsm', threshold=0.01)
attack_detection(model_name, net, test_data_loader, attack='advGAN', threshold=0.01)
attack_detection(model_name, net, test_data_loader, attack='advGAN_uni', threshold=0.01)
attack_detection(model_name, net, test_data_loader, attack='opt_uni', threshold=0.01)
attack_detection(model_name, net, test_data_loader, attack='opt', threshold=0.01)
def ex3_gen_adv(generator, gen_model, device):
root_dir = '../udacity-data'
adv_root_dir = '../udacity-data/adv_data'
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
image_size = (128, 128)
test_composed = transforms.Compose([Rescale(image_size), Preprocess(), ToTensor()])
basecnn = 'baseline.pt'
nvidia = 'nvidia.pt'
vgg16 = 'vgg16.pt'
model1 = BaseCNN()
model1.to(device)
model1.load_state_dict(torch.load(basecnn))
model1.eval()
model2 = Nvidia()
model2.to(device)
model2.load_state_dict(torch.load(nvidia))
model2.eval()
model3 = Vgg16()
model3.to(device)
model3.load_state_dict(torch.load(vgg16))
model3.eval()
target_models = []
target_models.append(('baseline', model1))
#target_models.append(('nvidia', model2))
target_models.append(('vgg16', model3))
train = 0
attacks = ['advGAN_attack']
# attacks = ['fgsm_attack', 'universal_attack', 'advGAN_attack', 'advGAN_universal_attack', 'opt_attack',]
target = 0.3
if train:
hmb_list =[('HMB1', 1479424215880976321),('HMB2', 1479424439139199216),('HMB4', 1479425729831388501), ('HMB5', 1479425834048269765), ('HMB6', 1479426202229245710)]
else:
hmb_list = [('testing', 1479425441182877835)]
# for (model_name, model) in target_models:
# noise_u = np.load(model_name + '_universal_attack_noise.npy')
# noise_u = torch.from_numpy(noise_u).type(torch.FloatTensor).to(device)
# advGAN_generator = Generator(3,3, model_name).to(device)
# advGAN_uni_generator = Generator(3,3, model_name).to(device)
# advGAN_generator.load_state_dict(torch.load('./models/' + model_name + '_netG_epoch_60.pth'))
# advGAN_uni_generator.load_state_dict(torch.load('./models/' + model_name + '_universal_netG_epoch_60.pth'))
# noise_seed = np.load(model_name + '_noise_seed.npy')
# noise_a = advGAN_uni_generator(torch.from_numpy(noise_seed).type(torch.FloatTensor).to(device))
# save_dir = os.path.join(adv_root_dir, model_name)
for (model_name, model) in target_models:
noise_u = np.load(model_name + '_universal_attack_noise.npy')
noise_u = torch.from_numpy(noise_u).type(torch.FloatTensor).to(device)
advGAN_generator = Generator(3,3, model_name).to(device)
advGAN_uni_generator = Generator(3,3, model_name).to(device)
advGAN_generator.load_state_dict(torch.load('./models/' + model_name + '_netG_epoch_60.pth'))
advGAN_generator.eval()
advGAN_uni_generator.load_state_dict(torch.load('./models/' + model_name + '_universal_netG_epoch_60.pth'))
advGAN_uni_generator.eval()
noise_seed = np.load(model_name + '_noise_seed.npy')
noise_a = advGAN_uni_generator(torch.from_numpy(noise_seed).type(torch.FloatTensor).to(device))
save_dir = os.path.join(adv_root_dir, model_name)
for (hmb, start) in hmb_list:
print(model_name ,hmb)
if train:
train_dataset = UdacityDataset(root_dir, [hmb], test_composed, type_='train')
else:
train_dataset = UdacityDataset(root_dir, [hmb], test_composed, type_='test')
generator = DataLoader(train_dataset, batch_size=64, shuffle=False, num_workers=8)
for i, batch in enumerate(generator):
batch_x = batch['image']
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
_, plt, _, perturbed_image = attack_test.fgsm_attack_test(model, batch_x, target, device, image_size=image_size)
plt.close()
if train:
for j in range(len(perturbed_image)):
np.save('../udacity-data/adv_data/' + model_name + '/fgsm_attack/' + hmb + '/' + str(i*64 + start + j), perturbed_image[j,:,:,:])
else:
np.save('../udacity-data/adv_testing/' + model_name + '/fgsm_attack/' + hmb + '/npy/' + 'batch_' + str(i), perturbed_image)
_, plt, perturbed_image = attack_test.optimized_uni_test(model, batch_x, device, noise_u, image_size=image_size)
plt.close()
if train:
for j in range(len(perturbed_image)):
np.save('../udacity-data/adv_data/' + model_name + '/universal_attack/' + hmb + '/' + str(i*64 + start + j), perturbed_image[j,:,:,:])
else:
np.save('../udacity-data/adv_testing/' + model_name + '/universal_attack/' + hmb + '/npy/' + 'batch_' + str(i), perturbed_image)
_, plt, perturbed_image = attack_test.advGAN_test(model, batch_x, advGAN_generator, device, image_size=image_size)
plt.close()
if train:
for j in range(len(perturbed_image)):
np.save('../udacity-data/adv_data/' + model_name + '/advGAN_attack/' + hmb + '/' + str(i*64 + start + j), perturbed_image[j,:,:,:])
else:
np.save('../udacity-data/adv_testing/' + model_name + '/advGAN_attack/' + hmb + '/npy/' + 'batch_' + str(i), perturbed_image)
_, plt, perturbed_image = attack_test.advGAN_uni_test(model, batch_x, device, noise_a, image_size=image_size)
plt.close()
if train:
for j in range(len(perturbed_image)):
np.save('../udacity-data/adv_data/' + model_name + '/advGAN_universal_attack/' + hmb + '/' + str(i*64 + start + j), perturbed_image[j,:,:,:])
else:
np.save('../udacity-data/adv_testing/' + model_name + '/advGAN_universal_attack/' + hmb + '/npy/' + 'batch_' + str(i), perturbed_image)
for (model_name, model) in target_models:
for (hmb, start) in hmb_list:
print(model_name, hmb)
if train:
train_dataset = UdacityDataset(root_dir, [hmb], test_composed, type_='train')
else:
train_dataset = UdacityDataset(root_dir, [hmb], test_composed, type_='test')
# npy = np.array([], dtype=np.float64).reshape(1, 3, 128, 128)
npy = None
for i in range(0, len(train_dataset)):
batch_x = train_dataset[i]['image']
batch_x = batch_x.unsqueeze(0)
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
_, plt, perturbed_image = attack_test.optimized_attack_test(model, batch_x, target, device, image_size=image_size)
plt.close()
if train:
for j in range(len(perturbed_image)):
np.save('../udacity-data/adv_data/' + model_name + '/opt_attack/' + hmb + '/' + str(i*64 + start + j), perturbed_image[j,:,:,:])
else:
if i == 0:
npy = perturbed_image
elif i % 64 != 0:
npy = np.concatenate((npy, perturbed_image))
else:
np.save('../udacity-data/adv_testing/' + model_name + '/opt_attack/' + hmb + '/npy/' + 'batch_' + str(i // 64 - 1), npy)
npy = perturbed_image
if not train:
np.save('../udacity-data/adv_testing/' + model_name + '/opt_attack/' + hmb + '/npy/' + 'batch_' + str(5614 // 64), npy)
def experiment_1():
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
target_models = []
basecnn = 'baseline.pt'
nvidia = 'nvidia.pt'
vgg16 = 'vgg16.pt'
model1 = BaseCNN()
model1.to(device)
model1.load_state_dict(torch.load(basecnn))
model1.eval()
model2 = Nvidia()
model2.to(device)
model2.load_state_dict(torch.load(nvidia))
model2.eval()
model3 = Vgg16()
model3.to(device)
model3.load_state_dict(torch.load(vgg16))
model3.eval()
target_models.append(('baseline', model1))
# target_models.append(('vgg16', model3))
# target_models.append(('nvidia', model2))
root_dir = '../udacity-data'
target = 0.3
attacks = ('FGSM', 'Optimization', 'Optimization Universal', 'AdvGAN', 'AdvGAN Universal')
fgsm_result = []
opt_result = []
optu_result = []
advGAN_result = []
advGANU_result = []
fgsm_diff = []
opt_diff = []
optu_diff = []
advGAN_diff = []
advGANU_diff = []
# models = ('baseline')
full_indices = list(range(5614))
test_indices = list(np.random.choice(5614, int(0.2*5614), replace=False))
train_indices = list(set(full_indices).difference(set(test_indices)))
image_size = (128, 128)
# if model_name == 'baseline':
# image_size = (128, 128)
# elif model_name == 'nvidia':
# image_size = (66, 200)
# elif model_name == 'vgg16':
# image_size = (224, 224)
test_composed = transforms.Compose([Rescale((image_size[1],image_size[0])), Preprocess(), ToTensor()])
# train_dataset = UdacityDataset(root_dir, ['HMB1', 'HMB2', 'HMB4'], test_composed, type_='train')
full_dataset = UdacityDataset(root_dir, ['testing'], test_composed, type_='test')
train_dataset = torch.utils.data.Subset(full_dataset, train_indices)
test_dataset = torch.utils.data.Subset(full_dataset, test_indices)
for (model_name, model) in target_models:
# train_size = int(0.8*len(full_dataset))
# test_size =len(full_dataset) - train_size
test_data_loader = torch.utils.data.DataLoader(full_dataset,batch_size=64,shuffle=False)
num_sample = len(full_dataset)
# universal perturbation generation
if not os.path.exists(model_name + '_universal_attack_noise.npy'):
print('Start universal attack training')
perturbation = generate_noise(train_dataset, model, model_name, device, target)
np.save(model_name + '_universal_attack_noise', perturbation)
print('Finish universal attack training.')
# # advGAN training
if not os.path.exists('./models/' + model_name + '_netG_epoch_60.pth'):
print('Start advGAN training')
advGAN = advGAN_Attack(model_name, model_name + '.pt', target + 0.2, train_dataset)
torch.save(advGAN.netG.state_dict(), './models/' + model_name +'_netG_epoch_60.pth')
print('Finish advGAN training')
# # advGAN_uni training
if not os.path.exists('./models/' + model_name + '_universal_netG_epoch_60.pth'):
print('Start advGAN_uni training')
advGAN_uni = advGAN_Attack(model_name, model_name + '.pt', target + 0.2, train_dataset, universal=True)
advGAN_uni.save_noise_seed(model_name + '_noise_seed.npy')
torch.save(advGAN_uni.netG.state_dict(), './models/' + model_name +'_universal_netG_epoch_60.pth')
print('Finish advGAN_uni training')
print("Testing: " + model_name)
#fgsm attack
fgsm_ast, diff = fgsm_ex(test_data_loader, model, model_name, target, device, num_sample, image_size)
print(fgsm_ast)
fgsm_result.append(fgsm_ast)
fgsm_diff.append(diff)
# # optimization attack
opt_ast, diff = opt_ex(test_dataset, model, model_name, target, device, num_sample, image_size)
print(opt_ast)
opt_result.append(opt_ast)
opt_diff.append(diff)
# optimized-based universal attack
optu_ast, diff = opt_uni_ex(test_data_loader, model, model_name, target, device, num_sample, image_size)
print(optu_ast)
optu_result.append(optu_ast)
optu_diff.append(diff)
# advGAN attack
advGAN_ast, diff = advGAN_ex(test_data_loader, model, model_name, target, device, num_sample, image_size)
print(advGAN_ast)
advGAN_result.append(advGAN_ast)
advGAN_diff.append(diff)
# advGAN_universal attack
advGANU_ast, diff = advGAN_uni_ex(test_data_loader, model, model_name, target, device, num_sample, image_size)
print(advGANU_ast)
advGANU_result.append(advGANU_ast)
advGANU_diff.append(diff)
def experiment_2(gen=True):
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
target_models = []
basecnn = 'baseline.pt'
nvidia = 'nvidia.pt'
vgg16 = 'vgg16.pt'
model1 = BaseCNN()
model1.to(device)
model1.load_state_dict(torch.load(basecnn))
model1.eval()
model2 = Nvidia()
model2.to(device)
model2.load_state_dict(torch.load(nvidia))
model2.eval()
model3 = Vgg16()
model3.to(device)
model3.load_state_dict(torch.load(vgg16))
model3.eval()
target_models.append(('baseline', model1))
target_models.append(('nvidia', model2))
target_models.append(('vgg16', model3))
root_dir = '../udacity-data'
target = 0.3
models = ('baseline', 'nvidia', 'vgg16')
# ex2_fun(target_models[0], target_models[1], device)
attacks = ('fgsm_attack', 'opt_attack', 'universal_attack', 'advGAN_attack', 'advGAN_universal_attack')
# blackbox test adv image from baseline on nvidia
print('No.1')
result = ex2_fun(target_models[0], target_models[1], device)
# plt_ = ex2_draw(result)
# plt_.title('Test ' + target_models[0][0] + ' adv_image on ' + target_models[1][0] + ' model')
# plt_.savefig('experiment_result/experiment_2/0-1.jpg')
# plt_.close()
print('No.2')
result = ex2_fun(target_models[0], target_models[2], device)
# plt_ = ex2_draw(result)
# plt_.title('Test ' + target_models[0][0] + ' adv_image on ' + target_models[2][0] + ' model')
# plt_.savefig('experiment_result/experiment_2/0-2.jpg')
# plt_.close()
print('No.3')
result = ex2_fun(target_models[1], target_models[0], device)
# plt_ = ex2_draw(result)
# plt_.title('Test ' + target_models[1][0] + ' adv_image on ' + target_models[0][0] + ' model')
# plt_.savefig('experiment_result/experiment_2/1-0.jpg')
# plt_.close()
print('No.4')
result = ex2_fun(target_models[1], target_models[2], device)
# plt_ = ex2_draw(result)
# plt_.title('Test ' + target_models[1][0] + ' adv_image on ' + target_models[2][0] + ' model')
# plt_.savefig('experiment_result/experiment_2/1-2.jpg')
# plt_.close()
print('No.5')
result = ex2_fun(target_models[2], target_models[0], device)
# plt_ = ex2_draw(result)
# plt_.title('Test ' + target_models[2][0] + ' adv_image on ' + target_models[0][0] + ' model')
# plt_.savefig('experiment_result/experiment_2/2-0.jpg')
# plt_.close()
print('No.6')
result = ex2_fun(target_models[2], target_models[1], device)
def exp1_fig():
model = BaseCNN()
model_name = 'baseline'
model.load_state_dict(torch.load('baseline.pt'))
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
model = model.to(device)
model.eval()
target = 0.3
image = imread(
'F:\\udacity-data\\testing\\center\\1479425441182877835.jpg')[200:, :]
image = imresize(image, (128, 128))
image = image / 255.
image = torch.from_numpy(image.transpose((2, 0, 1))).unsqueeze(0)
image =image.type(torch.FloatTensor)
image = image.to(device)
output = model(image)
print(output)
advGAN_generator = Generator(3, 3, model_name).to(device)
advGAN_uni_generator = Generator(3, 3, model_name).to(device)
# fgsm
_, perturbed_image_fgsm, _, adv_output_fgsm, noise_fgsm = fgsm_attack(model, image, target, device)
print('fgsm', adv_output_fgsm)
perturbed_image_fgsm = perturbed_image_fgsm.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
noise_fgsm = noise_fgsm.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
perturbed_image_fgsm = draw(perturbed_image_fgsm, adv_output_fgsm.item(), output.item())
perturbed_image_fgsm = imresize(perturbed_image_fgsm, (128, 128))
# opt
perturbed_image_opt, noise_opt, _, adv_output_opt = optimized_attack(
model, target, image, device)
perturbed_image_opt = perturbed_image_opt.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
print('opt', adv_output_opt)
noise_opt = noise_opt.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
perturbed_image_opt = draw(perturbed_image_opt, adv_output_opt.item(), output.item())
perturbed_image_opt = imresize(perturbed_image_opt, (128, 128))
# optu
noise_optu = np.load(model_name + '_universal_attack_noise.npy')
noise_optu = torch.from_numpy(noise_optu).type(torch.FloatTensor).to(device)
perturbed_image_optu = image + noise_optu
perturbed_image_optu = torch.clamp(perturbed_image_optu, 0, 1)
adv_output_optu = model(perturbed_image_optu)
print('universal', adv_output_optu)
perturbed_image_optu = perturbed_image_optu.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
noise_optu = noise_optu.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
perturbed_image_optu = draw(perturbed_image_optu, adv_output_optu.item(), output.item())
perturbed_image_optu = imresize(perturbed_image_optu, (128, 128))
# advGAN
advGAN_generator.load_state_dict(torch.load(
'./models/' + model_name + '_netG_epoch_60.pth'))
noise_advGAN = advGAN_generator(image)
perturbed_image_advGAN = image + torch.clamp(noise_advGAN, -0.3, 0.3)
perturbed_image_advGAN = torch.clamp(perturbed_image_advGAN, 0, 1)
adv_output_advGAN = model(perturbed_image_advGAN)
print('advGAN', adv_output_advGAN)
perturbed_image_advGAN = perturbed_image_advGAN.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
noise_advGAN = noise_advGAN.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
perturbed_image_advGAN = draw(perturbed_image_advGAN, adv_output_advGAN.item(), output.item())
perturbed_image_advGAN = imresize(perturbed_image_advGAN, (128, 128))
# advGAN_U
advGAN_uni_generator.load_state_dict(torch.load(
'./models/' + model_name + '_universal_netG_epoch_60.pth'))
noise_seed = np.load(model_name + '_noise_seed.npy')
noise_advGAN_U = advGAN_uni_generator(torch.from_numpy(
noise_seed).type(torch.FloatTensor).to(device))
perturbed_image_advGAN_U = image + torch.clamp(noise_advGAN_U, -0.3, 0.3)
perturbed_image_advGAN_U = torch.clamp(perturbed_image_advGAN_U, 0, 1)
adv_output_advGAN_U = model(perturbed_image_advGAN_U)
print('advGAN_uni', adv_output_advGAN_U)
perturbed_image_advGAN_U = perturbed_image_advGAN_U.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
noise_advGAN_U = noise_advGAN_U.squeeze(0).detach().cpu().numpy().transpose(1, 2, 0)
perturbed_image_advGAN_U = draw(perturbed_image_advGAN_U, adv_output_advGAN_U.item(), output.item())
perturbed_image_advGAN_U = imresize(perturbed_image_advGAN_U, (128, 128))
plt.subplot(2,5,1)
plt.imshow(perturbed_image_fgsm)
# plt.text(0.3, 0.3, 'y: %.4f' % output.item())
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,2)
plt.imshow(perturbed_image_opt)
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,3)
plt.imshow(perturbed_image_optu)
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,4)
plt.imshow(perturbed_image_advGAN)
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,5)
plt.imshow(perturbed_image_advGAN_U)
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,6)
plt.imshow(np.clip(noise_fgsm*5,0,1))
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,7)
plt.imshow(np.clip(noise_opt*5,0,1))
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,8)
plt.imshow(np.clip(noise_optu*5,0,1))
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,9)
plt.imshow(np.clip(noise_advGAN*5,0,1))
plt.xticks([])
plt.yticks([])
plt.subplot(2,5,10)
plt.imshow(np.clip(noise_advGAN_U*5,0,1))
plt.xticks([])
plt.yticks([])
plt.tight_layout(pad=0.5, w_pad=0, h_pad=0)
plt.show()
return
# def advGAN_test(model, image, generator, device):
def optu_test(model, image, noise, device):
perturbed_image_optu = image + noise
perturbed_image_optu = torch.clamp(perturbed_image_optu, 0, 1)
adv_output_optu = model(perturbed_image_optu)
perturbed_image_optu = perturbed_image_optu.squeeze(
0).detach().cpu().numpy().transpose(1, 2, 0)
if __name__ == "__main__":
model = BaseCNN()
model_name = 'baseline'
model.load_state_dict(torch.load('baseline.pt'))
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
model = model.to(device)
model.eval()
target = 0.3
# image = imread(
# '/media/dylan/Program/cg23-dataset/testing/center/1479425441182877835.jpg')[200:, :]
# image = imresize(image, (128, 128))
# image = image / 255.
# image = torch.from_numpy(image.transpose((2, 0, 1))).unsqueeze(0)
# image =image.type(torch.FloatTensor)
# image = image.to(device)
# output = model(image)
resized_image_width = 128
# if model_name == "baseline":
# resized_image_height = 128
# resized_image_width = 128
# elif model_name == "vgg16":
# resized_image_height = 224
# resized_image_width = 224
# elif model_name == "nvidia":
# resized_image_height = 66
# resized_image_width = 200
image_size = (resized_image_width, resized_image_height)
dataset_path = args.root_dir
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# print(device)
if model_name == 'baseline':
net = BaseCNN()
elif model_name == 'nvidia':
net = Nvidia()
elif model_name == 'vgg16':
net = Vgg16()
net.apply(weight_init)
net = net.to(device)
# net.to(device)
if train != 0:
if train == 2:
net.load_state_dict(torch.load(model_name + '.pt'))
composed = transforms.Compose([
Rescale(image_size),
RandFlip(),
test_composed = transforms.Compose(
[Rescale((128, 128)),
Preprocess('baseline'),
ToTensor()])
full_dataset = UdacityDataset(root_dir, ['testing'],
test_composed,
type_='test')
train_size = int(0.8 * len(full_dataset))
test_size = len(full_dataset) - train_size
train_dataset, test_dataset = torch.utils.data.random_split(
full_dataset, [train_size, test_size])
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
dataloader = torch.utils.data.DataLoader(train_dataset, 1, True)
model = BaseCNN()
model.to(device)
model.load_state_dict(torch.load(target_model))
model.eval()
perturbation = universal_attack(dataloader, model, device, target, 0.3)
np.save('universal_attack_noise.npy', perturbation.detach().cpu().numpy())
# i = 0
# test_dataloader = torch.utils.data.DataLoader(test_dataset,1,True)
# for _, data in enumerate(test_dataloader):
# image = data['image']
# perturbed_image = image + perturbation
# image =image.type(torch.FloatTensor)
# perturbed_image = perturbed_image.type(torch.FloatTensor)
# image = image.to(device)
# perturbed_image = perturbed_image.to(device)
def test_on_gen(net, model_name, dataset_path, attack, device):
original_net = None
if model_name == 'baseline':
original_net = BaseCNN()
elif model_name == 'nvidia':
original_net = Nvidia()
elif model_name == 'vgg16':
original_net = build_vgg16(False)
original_net.load_state_dict(torch.load(model_name + '.pt'))
original_net = original_net.to(device)
original_net.eval()
test_y = pd.read_csv('ch2_final_eval.csv')['steering_angle'].values
test_composed = transforms.Compose(
[Rescale(image_size),
Preprocess('baseline'),
ToTensor()])
test_dataset = UdacityDataset(dataset_path, ['testing'], test_composed,
'test')
test_generator = DataLoader(test_dataset, batch_size=1, shuffle=False)
with torch.no_grad():
# test on original dataset
yhat = []
y_original = []
# test_y = []
for _, sample_batched in enumerate(test_generator):
batch_x = sample_batched['image']
batch_y = sample_batched['steer']
batch_x = batch_x.type(torch.FloatTensor)
batch_y = batch_y.type(torch.FloatTensor)
batch_x = batch_x.to(device)
batch_y = batch_y.to(device)
output = net(batch_x)
output_ori = original_net(batch_x)
# print(output.item(), batch_y.item())
yhat.append(output.item())
y_original.append(output_ori.item())
yhat = np.array(yhat)
y_original = np.array(y_original)
rmse = np.sqrt(np.mean((yhat - test_y)**2))
rmse_ori = np.sqrt(np.mean((y_original - test_y)**2))
print('adv model on ori dataset:', rmse, 'ori model on ori dataset: ',
rmse_ori)
plt.figure(figsize=(32, 8))
plt.plot(test_y, 'r.-', label='target')
plt.plot(yhat, 'b^-', label='predict')
plt.legend(loc='best')
plt.title("RMSE: %.2f" % rmse)
# plt.show()
model_fullname = "%s_%d.png" % (model_name + '_' + attack,
int(time.time()))
plt.savefig(model_fullname)
test_generator = DataLoader(test_dataset, batch_size=64, shuffle=False)
target = 0.3
# test adv_training model on adv images generated based on itself
# ast_ori, _ = fgsm_ex(test_generator, original_net, 'baseline', target, device, len(test_dataset))
# ast_dist,_ = fgsm_ex(test_generator, net, 'baseline', target, device, len(test_dataset))
# print(ast_ori, ast_dist)
success = 0
success_ = 0
# test adv_training model on adv images generated based on original model
for _, sample_batched in enumerate(test_generator):
batch_x = sample_batched['image']
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
y_pred = net(batch_x)
y_pred_ori = original_net(batch_x)
# fgsm_attack
adv_x = fgsm_attack_(original_net, batch_x, target, device)
y_fgsm = net(adv_x)
y_ori_fgsm = original_net(adv_x)
diff = abs(y_fgsm - y_pred)
success += len(diff[diff >= abs(target)])
diff = abs(y_ori_fgsm - y_pred_ori)
success_ += len(diff[diff >= abs(target)])
print('fgsm', success / len(test_dataset), success_ / len(test_dataset))
# opt attack
# success = 0
# success_ = 0
# for _,sample_batched in enumerate(test_dataset):
# batch_x = sample_batched['image']
# batch_x = batch_x.type(torch.FloatTensor)
# batch_x = batch_x.unsqueeze(0)
# batch_x = batch_x.to(device)
# y_pred = net(batch_x)
# y_pred_ori = original_net(batch_x)
# # fgsm_attack
# # adv_x = fgsm_attack_(original_net, batch_x, target, device)
# adv_x,_,_,_ = optimized_attack(original_net, target, batch_x)
# y_fgsm = net(adv_x)
# y_ori_fgsm = original_net(adv_x)
# diff = abs(y_fgsm - y_pred)
# success += len(diff[diff >= abs(target)])
# diff = abs(y_ori_fgsm - y_pred_ori)
# success_ += len(diff[diff >= abs(target)])
# print('opt', success / len(test_dataset), success_ / len(test_dataset))
# opt universal attack
noise_u = np.load(model_name + '_universal_attack_noise.npy')
noise_u = torch.from_numpy(noise_u).type(torch.FloatTensor).to(device)
success = 0
success_ = 0
for _, sample_batched in enumerate(test_generator):
batch_x = sample_batched['image']
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
y_pred = net(batch_x)
y_pred_ori = original_net(batch_x)
# adv_x = fgsm_attack_(original_net, batch_x, target, device)
# noise = advGAN_generator(batch_x)
perturbed_image = batch_x + noise_u
adv_x = torch.clamp(perturbed_image, 0, 1)
y_fgsm = net(adv_x)
y_ori_fgsm = original_net(adv_x)
diff = abs(y_fgsm - y_pred)
success += len(diff[diff >= abs(target)])
diff = abs(y_ori_fgsm - y_pred_ori)
success_ += len(diff[diff >= abs(target)])
print('opt uni', success / len(test_dataset), success_ / len(test_dataset))
# test for advGAN attack
success = 0
success_ = 0
advGAN_generator = Generator(3, 3, model_name).to(device)
advGAN_generator.load_state_dict(
torch.load('./models/' + model_name + '_netG_epoch_60.pth'))
for _, sample_batched in enumerate(test_generator):
batch_x = sample_batched['image']
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
y_pred = net(batch_x)
y_pred_ori = original_net(batch_x)
# adv_x = fgsm_attack_(original_net, batch_x, target, device)
noise = advGAN_generator(batch_x)
perturbed_image = batch_x + torch.clamp(noise, -0.3, 0.3)
adv_x = torch.clamp(perturbed_image, 0, 1)
y_fgsm = net(adv_x)
y_ori_fgsm = original_net(adv_x)
diff = abs(y_fgsm - y_pred)
success += len(diff[diff >= abs(target)])
diff = abs(y_ori_fgsm - y_pred_ori)
success_ += len(diff[diff >= abs(target)])
print('advGAN', success / len(test_dataset), success_ / len(test_dataset))
# test for advGAN uni attack
advGAN_uni_generator = Generator(3, 3, model_name).to(device)
advGAN_uni_generator.load_state_dict(
torch.load('./models/' + model_name + '_universal_netG_epoch_60.pth'))
noise_seed = np.load(model_name + '_noise_seed.npy')
noise_a = advGAN_uni_generator(
torch.from_numpy(noise_seed).type(torch.FloatTensor).to(device))
success = 0
success_ = 0
for _, sample_batched in enumerate(test_generator):
batch_x = sample_batched['image']
batch_x = batch_x.type(torch.FloatTensor)
batch_x = batch_x.to(device)
y_pred = net(batch_x)
y_pred_ori = original_net(batch_x)
# adv_x = fgsm_attack_(original_net, batch_x, target, device)
# noise = advGAN_generator(batch_x)
perturbed_image = batch_x + torch.clamp(noise_a, -0.3, 0.3)
adv_x = torch.clamp(perturbed_image, 0, 1)
y_fgsm = net(adv_x)
y_ori_fgsm = original_net(adv_x)
diff = abs(y_fgsm - y_pred)
success += len(diff[diff >= abs(target)])
diff = abs(y_ori_fgsm - y_pred_ori)
success_ += len(diff[diff >= abs(target)])
print('advGAN uni', success / len(test_dataset),
success_ / len(test_dataset))
train_indices = list(set(full_indices).difference(set(test_indices)))
test_composed = transforms.Compose([
Rescale((image_size[1], image_size[0])),
Preprocess(),
ToTensor()
])
full_dataset = UdacityDataset(dataset_path, ['testing'],
test_composed,
type_='test')
train_dataset = torch.utils.data.Subset(full_dataset, train_indices)
test_dataset = torch.utils.data.Subset(full_dataset, test_indices)
for model_name in models_name:
if model_name == 'baseline':
net = BaseCNN()
elif model_name == 'nvidia':
net = Nvidia()
elif model_name == 'vgg16':
net = Vgg16()
net = net.to(device)
for attack in attacks:
print(model_name, attack)
adv_dataset_path = adv_datasets + '/' + model_name + '/' + attack
if not os.path.exists(adv_dataset_path):
os.mkdir(adv_dataset_path)
advt_model = model_name + '_' + attack
if train != 0:
if train == 2: