def ada_attack(model, img, max_iter, epsilon, mask):
P = [0.58395, 0.5712, 0.57375] #RGB
# 数据处理过程先转换为RGB，然后resize到800，然后正则化
data = prepare_data(model, img) # tuple类型
data['img'][0] = torch.autograd.Variable(data['img'][0],
requires_grad=True) # 需要可导
noise = np.zeros([800, 800, 3]) # 处理好图片的大小
finalimg = img
last_loss = []
for iter_n in range(max_iter):
loss = total_loss(model, data)
#print(loss)
if loss > 0:
loss.backward()
grad = data['img'][0].grad
grad_np = grad.data.cpu().numpy()
normalized_grad = np.sign(grad_np)
else:
return finalimg, NOISE
last_loss.append(loss.item())
temp_list = last_loss[-5:]
if temp_list[-1] > temp_list[0]:
if epsilon > 1:
epsilon = epsilon - 1.0
else:
epsilon = 1.0
NP_P = normalized_grad * epsilon # 整个输入图片的单次扰动
noise += NP_P.squeeze().transpose(1, 2, 0) # [800,800,3]
NOISE = cv2.resize(noise * P, (500, 500)) * mask
NOISE = NOISE[:, :, ::-1] # 从RGB对应的数据转换为BGR
finalimg = np.clip((img - NOISE), 0, 255) # 改为int32
finalimg = np.uint8(finalimg)
temp_data = prepare_data(model, finalimg)
data['img'][0].data = temp_data['img'][0].data
return finalimg, NOISE
def singlemask(img, model):
# 先获取有效分值的下坐标
with torch.no_grad():
data = prepare_data(model, img) # tuple类型
result = model(return_loss=False, rescale=True, **data) # 将rpn去掉
# 总共80个类别
index = []
for i in range(len(result)):
for j in range(len(result[i])):
if (result[i][j, 4] > 0.3):
index.append([i, j])
# 计算每个目标最多可以贴几个种子块（将三分之二的块设置为种子块）
object_num = np.min([len(index), 10])
Kflag = []
for i in range(object_num):
data = prepare_data(model, img) # tuple类型
data['img'][0].requires_grad = True # 需要可导
noise = np.zeros([800, 800, 3])
result = model(return_loss=False, rescale=True, **data) # 将rpn去掉
loss = result[index[i][0]][index[i][1], 4]
loss.backward()
grad = data['img'][0].grad
grad_np = grad.data.cpu().numpy()
normalized_grad = grad_np / np.sqrt(
np.sum(grad_np * grad_np, axis=(2, 3), keepdims=True)) # l2
NP_P = normalized_grad * 600 # 整个输入图片的单次扰动
noise += NP_P.squeeze().transpose(1, 2, 0) # [512,512,3]
NOISE = cv2.resize(noise, (500, 500))
Kflag.append(Norm_Sel_Mask(NOISE, 20))
return Kflag
def L2_attack(yolo_model, rcnn_model, img, conf_thresh, max_iter, epsilon,
mask):
P = [0.58395, 0.5712, 0.57375] # RGB
# yolo 部分
noise_yolo = np.zeros([608, 608, 3]) # 设置为原图大小
# 将输入图片转换到PIL格式
finalimg = Image.fromarray(cv2.cvtColor(
img, cv2.COLOR_BGR2RGB)) # 500*500*3 RGB
new_input = resize_big(finalimg)
# noise_yolo = 0.2*np.array(new_input,dtype=np.float)
# faster rcnn部分
idx = GetObject(rcnn_model, img)
data = prepare_data(rcnn_model, img) # tuple类型
data['img'][0].requires_grad = True # 需要可导
noise_rcnn = np.zeros([800, 800, 3]) # 处理好图片的大小
for iter_n in range(max_iter):
rloss = rcnn_L2_loss(rcnn_model, data, idx)
im, yloss = yolo_loss(yolo_model, new_input, conf_thresh, use_cuda=1)
tloss = 0.5 * yloss + 0.5 * rloss
print(tloss.item())
if tloss > 0:
# yolo
tloss.backward()
if (yloss > 0):
grad_yolo = im.grad
grad_np_yolo = grad_yolo.data.cpu().numpy()
normalized_grad_yolo = grad_np_yolo / np.sqrt(
np.sum(grad_np_yolo * grad_np_yolo,
axis=(2, 3),
keepdims=True)) # l2
else:
normalized_grad_yolo = np.zeros([1, 3, 608, 608])
# faster rcnn
if (rloss > 0):
grad_rcnn = data['img'][0].grad
grad_np_rcnn = grad_rcnn.data.cpu().numpy()
normalized_grad_rcnn = grad_np_rcnn / np.sqrt(
np.sum(grad_np_rcnn * grad_np_rcnn,
axis=(2, 3),
keepdims=True)) # l2
else:
normalized_grad_rcnn = np.zeros([1, 3, 800, 800])
else:
return finalimg, NOISE
NP_P_yolo = normalized_grad_yolo * epsilon # 整个输入图片的单次扰动
noise_yolo += NP_P_yolo.squeeze().transpose(1, 2, 0) # [608,608,3]
NP_P_rcnn = normalized_grad_rcnn * epsilon # faster rcnn 量级不一样
noise_rcnn += NP_P_rcnn.squeeze().transpose(1, 2, 0)
NOISE = cv2.resize(noise_yolo, (500, 500)) * mask + cv2.resize(
noise_rcnn * P, (500, 500)) * mask
temp_img = np.clip((img - NOISE[:, :, ::-1]), 0, 255) # 改为int32
X = np.uint8(temp_img)
finalimg = Image.fromarray(cv2.cvtColor(
X, cv2.COLOR_BGR2RGB)) # 500*500*3
new_input = resize_big(finalimg)
temp_data = prepare_data(rcnn_model, X)
data['img'][0].data = temp_data['img'][0].data
return finalimg, NOISE
def GetObject(model, img):
# 先获取faster_rcnn有效分值的下坐标
with torch.no_grad():
data = prepare_data(model, img) # tuple类型
result = model(return_loss=False, rescale=True, **data) # 将rpn去掉
# 总共80个类别
index = []
for i in range(len(result)):
for j in range(len(result[i])):
if (result[i][j, 4] > 0.3):
index.append([i, j])
return index
def L2_attack(model, img, max_iter, epsilon, mask):
P = [0.58395, 0.5712, 0.57375] #RGB
# 数据处理过程先转换为RGB，然后resize到800，然后正则化
data = prepare_data(model, img) # tuple类型
data['img'][0] = torch.autograd.Variable(data['img'][0],
requires_grad=True) # 需要可导
noise = np.zeros([500, 500, 3]) # 处理好图片的大小
finalimg = img
NOISE = np.zeros([500, 500, 3])
idx = GetObject(model, img)
for iter_n in range(max_iter):
loss = L2_loss(model, data, idx)
'''
if iter_n==0:
min_loss = loss # 当前最小的loss
if (loss <min_loss):
score = ((min_loss-loss)/min_loss).item()
min_loss = loss
NOISE += NP_P * mask * score # 加上前一次的噪声
'''
#print(loss)
if loss > 0:
loss.backward()
grad = data['img'][0].grad
grad_np = grad.data.cpu().numpy()
normalized_grad = grad_np / np.sqrt(
np.sum(grad_np * grad_np, axis=(2, 3), keepdims=True)) # l2
else:
return finalimg, NOISE
NP_P = normalized_grad * epsilon # 整个输入图片的单次扰动
NP_P = NP_P.squeeze().transpose(1, 2, 0)
NP_P = cv2.resize(NP_P, (500, 500))
noise += NP_P * mask # [800,800,3]
finalimg = np.clip((np.array(img, dtype=np.float) - noise[:, :, ::-1]),
0, 255) # 改为int32
NOISE = np.array(img, dtype=np.float) - finalimg
finalimg = np.uint8(finalimg)
temp_data = prepare_data(model, finalimg)
data['img'][0].data = temp_data['img'][0].data
return finalimg, NOISE
def gen_attack(model, img, norm_ord, max_iter, epsilon, mask):
P = [0.58395, 0.5712, 0.57375] #RGB
# 数据处理过程先转换为RGB，然后resize到800，然后正则化
data = prepare_data(model, img) # tuple类型
data['img'][0].requires_grad = True # 需要可导
noise = np.zeros([800, 800, 3]) # 处理好图片的大小
finalimg = img
for iter_n in range(max_iter):
loss = total_loss(model, data)
#print(loss)
if loss > 0:
loss.backward()
grad = data['img'][0].grad
grad_np = grad.data.cpu().numpy()
if norm_ord == 'sign':
normalized_grad = np.sign(grad_np)
elif norm_ord == 'L1':
normalized_grad = grad_np / np.sum(
np.abs(grad_np), axis=(2, 3), keepdims=True) # l1
elif norm_ord == 'L2':
normalized_grad = grad_np / np.sqrt(
np.sum(grad_np * grad_np, axis=(2, 3),
keepdims=True)) # l2
else:
raise ValueError('This norm_ord does not support...')
else:
return finalimg, NOISE
NP_P = normalized_grad * epsilon # 整个输入图片的单次扰动
noise += NP_P.squeeze().transpose(1, 2, 0) # [800,800,3]
NOISE = cv2.resize(noise * P, (500, 500)) * mask
NOISE = NOISE[:, :, ::-1] # 从RGB对应的数据转换为BGR
finalimg = np.clip((img - NOISE), 0, 255) # 改为int32
finalimg = np.uint8(finalimg)
temp_data = prepare_data(model, finalimg)
data['img'][0].data = temp_data['img'][0].data
return finalimg, NOISE
def ada_attack(yolo_model, rcnn_model, img, conf_thresh, max_iter, epsilon,
mask):
# yolo 部分
noise_yolo = np.zeros([608, 608, 3]) # 设置为原图大小
# 将输入图片转换到PIL格式
finalimg = Image.fromarray(cv2.cvtColor(
img, cv2.COLOR_BGR2RGB)) # 500*500*3 RGB
new_input = resize_big(finalimg)
# noise_yolo = 0.2*np.array(new_input,dtype=np.float)
# faster rcnn部分
data = prepare_data(rcnn_model, img) # tuple类型
data['img'][0].requires_grad = True # 需要可导
noise_rcnn = np.zeros([800, 800, 3]) # 处理好图片的大小
last_loss = []
NOISE = None
for iter_n in range(max_iter):
rloss = rcnn_loss(rcnn_model, data)
im, yloss = yolo_loss(yolo_model, new_input, conf_thresh, use_cuda=1)
tloss = (yloss + rloss) / 2 # faster_rcnn和yolo损失的融合
# print(tloss.item())
if tloss > 0:
# yolo
tloss.backward()
if yloss > 0:
grad_yolo = im.grad
grad_np_yolo = grad_yolo.data.cpu().numpy()
normalized_grad_yolo = np.sign(grad_np_yolo)
else:
normalized_grad_yolo = np.zeros([1, 3, 608, 608])
# faster rcnn
if rloss > 0:
grad_rcnn = data['img'][0].grad
grad_np_rcnn = grad_rcnn.data.cpu().numpy()
normalized_grad_rcnn = np.sign(grad_np_rcnn)
else:
normalized_grad_rcnn = np.zeros([1, 3, 800, 800])
else:
return finalimg, NOISE
last_loss.append(tloss.item())
temp_list = last_loss[-5:]
if temp_list[-1] > temp_list[0]:
if epsilon > 1:
epsilon = epsilon * 0.93
else:
epsilon = 1.0
NP_P_yolo = normalized_grad_yolo * epsilon # 整个输入图片的单次扰动
noise_yolo += NP_P_yolo.squeeze().transpose(1, 2, 0) # [608,608,3]
NP_P_rcnn = normalized_grad_rcnn * epsilon
noise_rcnn += NP_P_rcnn.squeeze().transpose(1, 2, 0)
NOISE = cv2.resize(noise_yolo, (500, 500)) * mask + cv2.resize(
noise_rcnn, (500, 500)) * mask
ind_one = np.logical_and(NOISE >= 0, NOISE <= 1.1)
ind_minus_one = np.logical_and(NOISE >= -1.1, NOISE < 0)
NOISE[ind_one] = 1.1
NOISE[ind_minus_one] = -1.1
NOISE = NOISE * mask
if iter_n == 0:
NOISE += np.ones_like(NOISE)
temp_img = np.clip((img - NOISE[:, :, ::-1]), 0, 255) # 改为int32
X = np.uint8(temp_img)
finalimg = Image.fromarray(cv2.cvtColor(
X, cv2.COLOR_BGR2RGB)) # 500*500*3
new_input = resize_big(finalimg)
temp_data = prepare_data(rcnn_model, X)
data['img'][0].data = temp_data['img'][0].data
return finalimg, NOISE
