def PGD_attack_mtask(x, y, mask, net, criterion, task_name, epsilon, steps, dataset, step_size, info, args, using_noise=True):
net.eval()
# tensor_std = get_torch_std(info)
if epsilon == 0:
return Variable(x, requires_grad=False)
GPU_flag = False
if torch.cuda.is_available():
GPU_flag=True
rescale_term = 2./255
epsilon = epsilon * rescale_term
step_size = step_size * rescale_term
# print('epsilon', epsilon, epsilon / rescale_term)
x_adv = x.clone()
pert_upper = x_adv + epsilon
pert_lower = x_adv - epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = -torch.ones_like(x_adv)
# lower_bound = torch.zeros_like(x_adv) #bug
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
#TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
for keys, m in mask.items():
mask[keys] = m.cuda()
for keys, tar in y.items():
y[keys] = tar.cuda()
if using_noise:
noise = torch.FloatTensor(x.size()).uniform_(-epsilon, epsilon)
if GPU_flag:
noise = noise.cuda()
x_adv = x_adv + noise
x_adv = clamp_tensor(x_adv, lower_bound, upper_bound)
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
h_adv = net(x_adv)
grad_total_loss = None
for each in task_name:
if grad_total_loss is None:
grad_total_loss = criterion[each](h_adv[each], y[each], mask[each])
else:
grad_total_loss = grad_total_loss + criterion[each](h_adv[each], y[each], mask[each])
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
grad_total_loss.backward()
x_adv.grad.sign_()
x_adv = x_adv + step_size * x_adv.grad
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
x_adv = Variable(x_adv.data, requires_grad=True) #TODO: optimize, remove this variable init each
#TODO: volatile option for backward, check later
# sample =x_adv.data
# im_rgb = np.moveaxis(sample[1].cpu().numpy().squeeze(), 0, 2)
#
# import matplotlib.pyplot as plt
# plt.imshow(im_rgb)
# plt.show()
return x_adv
def PGD_attack(x, y, net, Loss, epsilon, steps, dataset, step_size, info, using_noise=True):
# print('crop ', torch.max(x), torch.min(x))
# print('size', x.size())
std_array = np.asarray(info["std"])
tensor_std = torch.from_numpy(std_array)
tensor_std = tensor_std.unsqueeze(0)
tensor_std = tensor_std.unsqueeze(2)
tensor_std = tensor_std.unsqueeze(2).float()
GPU_flag = False
if torch.cuda.is_available():
GPU_flag=True
x_adv = x.clone()
pert_epsilon = torch.ones_like(x_adv) * epsilon / tensor_std
pert_upper = x_adv + pert_epsilon
pert_lower = x_adv - pert_epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = torch.zeros_like(x_adv)
upper_bound = forward_transform(upper_bound, info)
lower_bound = forward_transform(lower_bound, info)
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
#TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
Loss = Loss.cuda()
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
tensor_std = tensor_std.cuda()
ones_x = ones_x.cuda()
y = y.cuda()
step_size_tensor = ones_x * step_size / tensor_std
if using_noise:
noise = torch.FloatTensor(x.size()).uniform_(-epsilon, epsilon)
if GPU_flag:
noise = noise.cuda()
noise = noise / tensor_std
x_adv = x_adv + noise
x_adv = clamp_tensor(x_adv, lower_bound, upper_bound)
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
h_adv = net(x_adv)
# elif dataset == 'ade20k':
# h_adv = net(x_adv,segSize = (256,256))
cost = Loss(h_adv[0], y) #TODO: works, but is this the correct place to convert to long??
#print(str(i) + ': ' + str(cost.data))
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
cost.backward()
x_adv.grad.sign_()
x_adv = x_adv + step_size_tensor * x_adv.grad
#print(x_adv.data[:,4,4])
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
# x_adv = torch.where(x_adv > upper_bound, upper_bound, x_adv)
# x_adv = torch.where(x_adv < lower_bound, lower_bound, x_adv)
x_adv = Variable(x_adv.data, requires_grad=True) #TODO: optimize, remove this variable init each
#TODO: volatile option for backward, check later
return x_adv
def PGD_masked_attack_mtask_city(x, y, mask, attack_mask, net, criterion, task_name, epsilon, steps, dataset, step_size, info, args, using_noise=True):
# print('crop ', torch.max(x), torch.min(x))
# print('size', x.size())
# std_array = np.asarray(info["std"])
# tensor_std = torch.from_numpy(std_array)
# tensor_std = tensor_std.unsqueeze(0)
# tensor_std = tensor_std.unsqueeze(2)
# tensor_std = tensor_std.unsqueeze(2).float()
tensor_std = get_torch_std(info)
GPU_flag = False
if torch.cuda.is_available():
GPU_flag=True
x_adv = x.clone()
epsilon = epsilon / 255.
step_size = step_size / 255.
ones_like_x_adv = torch.ones_like(x_adv)
if GPU_flag:
ones_like_x_adv = ones_like_x_adv.cuda()
tensor_std = tensor_std.cuda()
ones_like_x_adv = torch.ones_like(x_adv)
if GPU_flag:
ones_like_x_adv = ones_like_x_adv.cuda()
pert_epsilon = ones_like_x_adv * epsilon / tensor_std
pert_upper = x_adv + pert_epsilon
pert_lower = x_adv - pert_epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = torch.zeros_like(x_adv)
upper_bound = forward_transform(upper_bound, info)
lower_bound = forward_transform(lower_bound, info)
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
#TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
Loss = 0
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
tensor_std = tensor_std.cuda()
ones_x = ones_x.cuda()
for keys, m in mask.items():
mask[keys] = m.cuda()
for keys, tar in y.items():
y[keys] = tar.cuda()
step_size_tensor = ones_x * step_size / tensor_std
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
h_adv = net(x_adv) # dict{rep:float32,segmentasemantic:float32, depth_zbuffer:float32, reconstruct:float32}
grad_total_loss = None
# print("Task names ", task_name)
for each in task_name:
# print("IN ",each)
if grad_total_loss is None:
# print(each,y.keys(),h_adv[1])
# print(h_adv)
ignore_value = 255
# print(mask[each].type(), attack_mask.type())
attack_mask = attack_mask.long()
mask_each = mask[each] #segmentsemantic is long and others are float.
mask_total = mask_each * attack_mask # attack_mask is float, mask_total is float.
mask_total = mask_total.long()
# print(each, (y[each] * mask_total).type())
# print((ignore_value * (1-mask_total)).type()) # types(str, )
y[each] = y[each] * mask_total + ignore_value * (1-mask_total) # y is {auto:float,segsem:int64,deoth:float}
grad_total_loss = criterion[each](h_adv[each], y[each], mask[each]*attack_mask)
else:
grad_total_loss = grad_total_loss + criterion[each](h_adv[each], y[each], mask[each]*attack_mask)
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
grad_total_loss.backward()
x_adv.grad.sign_()
x_adv = x_adv + step_size_tensor * x_adv.grad
#print(x_adv.data[:,4,4])
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
# x_adv = torch.where(x_adv > upper_bound, upper_bound, x_adv)
# x_adv = torch.where(x_adv < lower_bound, lower_bound, x_adv)
x_adv = Variable(x_adv.data, requires_grad=True) #TODO: optimize, remove this variable init each
#TODO: volatile option for backward, check later
return x_adv
def PGD_drnseg_masked_attack_city(x, y, attack_mask, net, criterion, epsilon, steps, dataset, step_size, info, args, using_noise=True):
tensor_std = get_torch_std(info)
GPU_flag = False
if torch.cuda.is_available():
GPU_flag = True
x_adv = x.clone()
epsilon = epsilon / 255.
step_size = step_size / 255.
ones_like_x_adv = torch.ones_like(x_adv)
if GPU_flag:
ones_like_x_adv = ones_like_x_adv.cuda()
tensor_std = tensor_std.cuda()
pert_epsilon = ones_like_x_adv * epsilon / tensor_std
pert_upper = x_adv + pert_epsilon
pert_lower = x_adv - pert_epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = torch.zeros_like(x_adv)
upper_bound = forward_transform(upper_bound, info)
lower_bound = forward_transform(lower_bound, info)
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
# TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
# Loss = 0
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
tensor_std = tensor_std.cuda()
ones_x = ones_x.cuda()
y = y.cuda()
# for keys, m in mask.items():
# mask[keys] = m.cuda()
# for keys, tar in y.items():
# y[keys] = tar.cuda()
step_size_tensor = ones_x * step_size / tensor_std
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
#if i==10:
# print('PGD_drnseg_masked_attack_city attack step', i)
h_adv = net(x_adv) # dict{rep:float32,segmentasemantic:float32, depth_zbuffer:float32, reconstruct:float32}
grad_total_loss = None
# print("Task names ", task_name)
# for each in task_name:
# print("IN ",each)
# if grad_total_loss is None:
# print(each,y.keys(),h_adv[1])
# print(h_adv)
ignore_value = 255
# print(mask[each].type(), attack_mask.type())
attack_mask = attack_mask.long()
# mask_each = mask[each] # segmentsemantic is long and others are float.
# mask_total = mask_each * attack_mask # attack_mask is float, mask_total is float.
# mask_total = mask_total.long()
# print(each, (y[each] * mask_total).type())
# print((ignore_value * (1-mask_total)).type()) # types(str, )
y = y * attack_mask + ignore_value * (1 - attack_mask) # y is {auto:float,segsem:int64,deoth:float}
grad_total_loss = criterion(h_adv[0], y)
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
grad_total_loss.backward()
x_adv.grad.sign_()
x_adv = x_adv + step_size_tensor * x_adv.grad
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
x_adv = Variable(x_adv.data, requires_grad=True) # TODO: optimize, remove this variable init each
return x_adv
def PGD_attack_mtask_city(x, y, mask, net, criterion, task_name, epsilon, steps, dataset, step_size, info, args, using_noise=True):
# print('crop ', torch.max(x), torch.min(x))
# print('size', x.size())
# std_array = np.asarray(info["std"])
# tensor_std = torch.from_numpy(std_array)
# tensor_std = tensor_std.unsqueeze(0)
# tensor_std = tensor_std.unsqueeze(2)
# tensor_std = tensor_std.unsqueeze(2).float()
tensor_std = get_torch_std(info)
GPU_flag = False
if torch.cuda.is_available():
GPU_flag=True
x_adv = x.clone()
epsilon = epsilon / 255.
step_size = step_size / 255.
pert_epsilon = torch.ones_like(x_adv) * epsilon / tensor_std
pert_upper = x_adv + pert_epsilon
pert_lower = x_adv - pert_epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = torch.zeros_like(x_adv)
upper_bound = forward_transform(upper_bound, info)
lower_bound = forward_transform(lower_bound, info)
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
#TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
Loss = 0
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
tensor_std = tensor_std.cuda()
ones_x = ones_x.cuda()
for keys, m in mask.items():
mask[keys] = m.cuda()
for keys, tar in y.items():
y[keys] = tar.cuda()
step_size_tensor = ones_x * step_size / tensor_std
if using_noise:
noise = torch.FloatTensor(x.size()).uniform_(-epsilon, epsilon)
if GPU_flag:
noise = noise.cuda()
noise = noise / tensor_std
x_adv = x_adv + noise
x_adv = clamp_tensor(x_adv, lower_bound, upper_bound)
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
h_adv = net(x_adv)
grad_total_loss = None
for each in task_name:
if grad_total_loss is None:
grad_total_loss = criterion[each](h_adv[each], y[each], mask[each])
else:
grad_total_loss = grad_total_loss + criterion[each](h_adv[each], y[each], mask[each])
# # elif dataset == 'ade20k':
# # h_adv = net(x_adv,segSize = (256,256))
#
# # total_loss = 0
# # for keys, loss_func in criterion:
# # if keys in task_name:
# # loss = loss_func(h_adv[keys], y[keys])
# # total_loss += loss
#
# first_loss = None
# loss_dict = {}
# for c_name, criterion_fun in criterion.items():
# if first_loss is None:
# first_loss = c_name
# # print('l output target', output)
# # print('ratget', target)
# loss_dict[c_name] = criterion_fun(h_adv, y)
# # print('caname', c_name, loss_dict[c_name])
# else:
# loss_dict[c_name] = criterion_fun(h_adv[c_name], y[c_name])
#
# grad_total_loss = None
# for each in args.test_task_set:
# if grad_total_loss is None:
# grad_total_loss = loss_dict[each]
# else:
# grad_total_loss = grad_total_loss + loss_dict[each]
# cost = Loss(h_adv[0], y) #TODO: works, but is this the correct place to convert to long??
#print(str(i) + ': ' + str(cost.data))
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
grad_total_loss.backward()
x_adv.grad.sign_()
x_adv = x_adv + step_size_tensor * x_adv.grad
#print(x_adv.data[:,4,4])
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
# x_adv = torch.where(x_adv > upper_bound, upper_bound, x_adv)
# x_adv = torch.where(x_adv < lower_bound, lower_bound, x_adv)
x_adv = Variable(x_adv.data, requires_grad=True) #TODO: optimize, remove this variable init each
#TODO: volatile option for backward, check later
# sample =x_adv.data
# im_rgb = back_transform(sample, info)[0]
# im_rgb = np.moveaxis(im_rgb.cpu().numpy().squeeze(), 0, 2)
#
#
# import matplotlib.pyplot as plt
# plt.imshow(im_rgb)
# plt.show()
return x_adv
def PGD_attack_ensemble_mtask(x,
y,
mask,
net,
criterion_list,
task_name,
epsilon,
steps,
step_size,
using_noise=True,
momentum=False,
use_houdini=False):
net.eval()
# tensor_std = get_torch_std(info)
if epsilon == 0:
return Variable(x, requires_grad=False)
GPU_flag = False
if torch.cuda.is_available():
GPU_flag = True
rescale_term = 2. / 255
epsilon = epsilon * rescale_term
step_size = step_size * rescale_term
# print('epsilon', epsilon, epsilon / rescale_term)
x_adv = x.clone()
# print('x range', torch.min(x_adv), torch.max(x_adv))
pert_upper = x_adv + epsilon
pert_lower = x_adv - epsilon
upper_bound = torch.ones_like(x_adv)
lower_bound = -torch.ones_like(
x_adv) ####Big bug! used to be zero, totally wrong
upper_bound = torch.min(upper_bound, pert_upper)
lower_bound = torch.max(lower_bound, pert_lower)
#TODO: print and check the bound
ones_x = torch.ones_like(x).float()
if GPU_flag:
x_adv = x_adv.cuda()
upper_bound = upper_bound.cuda()
lower_bound = lower_bound.cuda()
for keys, m in mask.items():
mask[keys] = m.cuda()
for keys, tar in y.items():
y[keys] = tar.cuda()
if using_noise:
noise = torch.FloatTensor(x.size()).uniform_(-epsilon, epsilon)
if GPU_flag:
noise = noise.cuda()
x_adv = x_adv + noise
x_adv = clamp_tensor(x_adv, lower_bound, upper_bound)
x_adv = Variable(x_adv, requires_grad=True)
for i in range(steps):
h_adv = net(x_adv)
grad_total_loss = None
if use_houdini: houdini = Houdini.apply
for sub_output, sub_criteria in zip(h_adv, criterion_list):
for each in task_name:
if grad_total_loss is None:
if use_houdini:
task_loss = calc_iou(sub_output[each].float(),
y[each].long().squeeze(dim=1))
task_loss = task_loss.reshape(-1, 1, 1, 1).repeat(
1, 1, sub_output[each].shape[2],
sub_output[each].shape[3])
h_loss = houdini(sub_output[each].float(),
y[each].long(), task_loss, 255)
grad_total_loss = h_loss
else:
grad_total_loss = sub_criteria[each](sub_output[each],
y[each],
mask[each])
else:
if use_houdini:
task_loss = calc_iou(sub_output[each],
y[each].long().squeeze(dim=1))
h_loss = houdini(sub_output[each].float(),
y[each].long(), task_loss, 255)
grad_total_loss = grad_total_loss + h_loss
else:
grad_total_loss = grad_total_loss + sub_criteria[each](
sub_output[each], y[each], mask[each])
net.zero_grad()
if x_adv.grad is not None:
x_adv.grad.data.fill_(0)
grad_total_loss.backward()
if momentum:
if i == 0:
grad = x_adv.grad
old_grad = grad.clone()
else:
grad = old_grad * 0.5 + x_adv.grad
old_grad = grad.clone()
else:
grad = x_adv.grad
grad.sign_()
x_adv = x_adv + step_size * grad #x_adv.grad #TODO: seems previously we may have bug?
x_adv = clamp_tensor(x_adv, upper_bound, lower_bound)
x_adv = Variable(x_adv.data, requires_grad=True
) #TODO: optimize, remove this variable init each
#TODO: volatile option for backward, check later
# sample = x_adv.data
# im_rgb = np.moveaxis(sample[1].cpu().numpy().squeeze(), 0, 2)
#
# import matplotlib.pyplot as plt
# plt.imshow(im_rgb)
# plt.show()
return x_adv