def _gradient(self, inputs, labels):
"""
Calculate the gradient of input samples.
Args:
inputs (Union[numpy.ndarray, tuple]): Input samples.
labels (Union[numpy.ndarray, tuple]): Original/target labels. \
For each input if it has more than one label, it is wrapped in a tuple.
Returns:
numpy.ndarray, gradient of labels w.r.t inputs.
Examples:
>>> grad = self._gradient([[0.5, 0.3, 0.4]],
>>> [[0, 0, 0, 1, 0, 0, 0, 0, 0, 0])
"""
# get grad of loss over x
inputs_tensor = to_tensor_tuple(inputs)
labels_tensor = to_tensor_tuple(labels)
out_grad = self._loss_grad(*inputs_tensor, *labels_tensor)
if isinstance(out_grad, tuple):
out_grad = out_grad[0]
gradient = out_grad.asnumpy()
if self._is_targeted:
gradient = -gradient
return normalize_value(gradient, self._norm_level)
def _gradient(self, inputs, labels):
"""
Calculate gradients based on input samples and original/target labels.
Args:
inputs (Union[numpy.ndarray, tuple]): Input sample.
labels (Union[numpy.ndarray, tuple]): Original/target labels. \
For each input if it has more than one label, it is wrapped in a tuple.
Returns:
numpy.ndarray, gradient of inputs.
"""
inputs_tensor = to_tensor_tuple(inputs)
labels_tensor = to_tensor_tuple(labels)
out_grad = self._grad_all(*inputs_tensor, *labels_tensor)
if isinstance(out_grad, tuple):
out_grad = out_grad[0]
gradient = out_grad.asnumpy()
if self._is_targeted:
gradient = -gradient
return normalize_value(gradient, self._norm_level)
def _deepfool_detection_scores(inputs, gt_boxes, gt_labels, network):
"""
Evaluate the detection result of inputs, specially for object detection models.
Args:
inputs (numpy.ndarray): Input samples.
gt_boxes (numpy.ndarray): Ground-truth boxes of inputs.
gt_labels (numpy.ndarray): Ground-truth labels of inputs.
model (BlackModel): Target model.
Returns:
- numpy.ndarray, detection scores of inputs.
- numpy.ndarray, the number of objects that are correctly detected.
"""
network = check_param_type('network', network, Cell)
inputs_tensor = to_tensor_tuple(inputs)
box_and_confi, pred_logits = network(*inputs_tensor)
box_and_confi, pred_logits = box_and_confi.asnumpy(), pred_logits.asnumpy()
pred_labels = np.argmax(pred_logits, axis=2)
det_scores = []
correct_labels_num = []
gt_boxes_num = gt_boxes.shape[1]
iou_thres = 0.5
for idx, (boxes, labels) in enumerate(zip(box_and_confi, pred_labels)):
score = 0
box_num = boxes.shape[0]
gt_boxes_idx = gt_boxes[idx]
gt_labels_idx = gt_labels[idx]
correct_label_flag = np.zeros(gt_labels_idx.shape)
for i in range(box_num):
pred_box = boxes[i]
max_iou_confi = 0
for j in range(gt_boxes_num):
iou = calculate_iou(pred_box[:4], gt_boxes_idx[j][:4])
if labels[i] == gt_labels_idx[j] and iou > iou_thres:
max_iou_confi = max(max_iou_confi, pred_box[-1] + iou)
correct_label_flag[j] = 1
score += max_iou_confi
det_scores.append(score)
correct_labels_num.append(np.sum(correct_label_flag))
return np.array(det_scores), np.array(correct_labels_num)
def _generate_detection(self, inputs, labels):
"""Generate adversarial examples in detection scenario"""
images, auxiliary_inputs = inputs[0], inputs[1:]
gt_boxes, gt_labels = labels
_, gt_object_nums = _deepfool_detection_scores(inputs, gt_boxes,
gt_labels,
self._network)
if not self._sparse:
gt_labels = np.argmax(gt_labels, axis=2)
origin_labels = np.zeros(gt_labels.shape[0])
for i in range(gt_labels.shape[0]):
origin_labels[i] = np.argmax(np.bincount(gt_labels[i]))
images_dtype = images.dtype
iteration = 0
num_boxes = gt_labels.shape[1]
merge_net = _GetLogits(self._network)
detection_net_grad = GradWrap(merge_net)
weight = np.squeeze(np.zeros(images.shape[1:]))
r_tot = np.zeros(images.shape)
x_origin = images
while not _is_success((images,) + auxiliary_inputs, gt_boxes, gt_labels, self._network, gt_object_nums, \
self._reserve_ratio) and iteration < self._max_iters:
preds_logits = merge_net(
*to_tensor_tuple(images),
*to_tensor_tuple(auxiliary_inputs)).asnumpy()
grads = jacobian_matrix_for_detection(
detection_net_grad, (images, ) + auxiliary_inputs, num_boxes,
self._num_classes)
for idx in range(images.shape[0]):
diff_w = np.inf
label = int(origin_labels[idx])
auxiliary_input_i = tuple()
for item in auxiliary_inputs:
auxiliary_input_i += (np.expand_dims(item[idx], axis=0), )
gt_boxes_i = np.expand_dims(gt_boxes[idx], axis=0)
gt_labels_i = np.expand_dims(gt_labels[idx], axis=0)
inputs_i = (np.expand_dims(images[idx],
axis=0), ) + auxiliary_input_i
if _is_success(inputs_i, gt_boxes_i, gt_labels_i,
self._network, gt_object_nums[idx],
self._reserve_ratio):
continue
for k in range(self._num_classes):
if k == label:
continue
w_k = grads[k, idx, ...] - grads[label, idx, ...]
f_k = np.mean(
np.abs(preds_logits[idx, :, k, ...] -
preds_logits[idx, :, label, ...]))
if self._norm_level == 2 or self._norm_level == '2':
diff_w_k = abs(f_k) / (np.linalg.norm(w_k) + 1e-8)
elif self._norm_level == np.inf \
or self._norm_level == 'inf':
diff_w_k = abs(f_k) / (np.linalg.norm(w_k, ord=1) +
1e-8)
else:
msg = 'ord {} is not available.' \
.format(str(self._norm_level))
LOGGER.error(TAG, msg)
raise NotImplementedError(msg)
if diff_w_k < diff_w:
diff_w = diff_w_k
weight = w_k
if self._norm_level == 2 or self._norm_level == '2':
r_i = diff_w * weight / (np.linalg.norm(weight) + 1e-8)
elif self._norm_level == np.inf or self._norm_level == 'inf':
r_i = diff_w*np.sign(weight) \
/ (np.linalg.norm(weight, ord=1) + 1e-8)
else:
msg = 'ord {} is not available in normalization,' \
.format(str(self._norm_level))
LOGGER.error(TAG, msg)
raise NotImplementedError(msg)
r_tot[idx, ...] = r_tot[idx, ...] + r_i
if self._bounds is not None:
clip_min, clip_max = self._bounds
images = x_origin + (1 + self._overshoot) * r_tot * (clip_max -
clip_min)
images = np.clip(images, clip_min, clip_max)
else:
images = x_origin + (1 + self._overshoot) * r_tot
iteration += 1
images = images.astype(images_dtype)
del preds_logits, grads
return images