def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
# APRICOT inputs should have shape (1, None, None, 3) while DAPRICOT inputs have shape
# (3, None, None, 3)
images = tf.placeholder(tf.float32,
shape=(model_kwargs.get("batch_size",
1), None, None, 3))
model = TensorFlowFasterRCNN(
images,
model=None,
filename="faster_rcnn_resnet50_coco_2018_01_28",
url=
"http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_coco_2018_01_28.tar.gz",
sess=None,
is_training=False,
clip_values=(0, 1),
channels_first=False,
preprocessing_defences=None,
postprocessing_defences=None,
attack_losses=(
"Loss/RPNLoss/localization_loss",
"Loss/RPNLoss/objectness_loss",
"Loss/BoxClassifierLoss/localization_loss",
"Loss/BoxClassifierLoss/classification_loss",
),
)
return model
def test_compute_loss(art_warning, get_mnist_dataset):
try:
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
images = tf.placeholder(tf.float32, shape=[2, 28, 28, 3])
# Get test data
(_, _), (x_test_mnist, y_test_mnist) = get_mnist_dataset
frcnn = TensorFlowFasterRCNN(
images=images,
clip_values=(0, 1),
attack_losses=(
"Loss/RPNLoss/localization_loss",
"Loss/RPNLoss/objectness_loss",
"Loss/BoxClassifierLoss/localization_loss",
"Loss/BoxClassifierLoss/classification_loss",
),
)
# Create labels
result = frcnn.predict(
np.repeat(x_test_mnist[:2].astype(np.float32), repeats=3, axis=3))
y = [
{
"boxes": result[0]["boxes"],
"labels": result[0]["labels"],
"scores": np.ones_like(result[0]["labels"]),
},
{
"boxes": result[1]["boxes"],
"labels": result[1]["labels"],
"scores": np.ones_like(result[1]["labels"]),
},
]
# Compute loss
loss = frcnn.compute_loss(
np.repeat(x_test_mnist[:2].astype(np.float32), repeats=3, axis=3),
y)
assert float(loss) == pytest.approx(6.592308044433594, abs=0.5)
except ARTTestException as e:
art_warning(e)
def setUpClass(cls):
master_seed(seed=1234, set_tensorflow=True)
super().setUpClass()
cls.n_test = 10
cls.x_test_mnist = cls.x_test_mnist[0:cls.n_test]
cls.y_test_mnist = cls.y_test_mnist[0:cls.n_test]
# Only import if object detection module is available
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
# Define object detector
images = tf.placeholder(tf.float32, shape=[2, 28, 28, 1])
cls.obj_dec = TensorFlowFasterRCNN(images=images)
def test_errors(art_warning, get_mnist_dataset):
try:
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
images = tf.placeholder(tf.float32, shape=[1, 28, 28, 1])
with pytest.raises(ValueError):
TensorFlowFasterRCNN(
images=images,
clip_values=(1, 2),
attack_losses=("loss_classifier", "loss_box_reg",
"loss_objectness", "loss_rpn_box_reg"),
)
with pytest.raises(ValueError):
TensorFlowFasterRCNN(
images=images,
clip_values=(-1, 1),
attack_losses=("loss_classifier", "loss_box_reg",
"loss_objectness", "loss_rpn_box_reg"),
)
from art.defences.postprocessor.rounded import Rounded
post_def = Rounded()
with pytest.raises(ValueError):
TensorFlowFasterRCNN(
images=images,
clip_values=(0, 1),
attack_losses=("loss_classifier", "loss_box_reg",
"loss_objectness", "loss_rpn_box_reg"),
postprocessing_defences=post_def,
)
except ARTTestException as e:
art_warning(e)
def test_preprocessing_defences(art_warning, get_mnist_dataset):
try:
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
from art.defences.preprocessor.spatial_smoothing import SpatialSmoothing
images = tf.placeholder(tf.float32, shape=[1, 28, 28, 3])
pre_def = SpatialSmoothing()
with pytest.raises(ValueError):
_ = TensorFlowFasterRCNN(
images=images,
clip_values=(0, 1),
attack_losses=("loss_classifier", "loss_box_reg",
"loss_objectness", "loss_rpn_box_reg"),
preprocessing_defences=pre_def,
)
except ARTTestException as e:
art_warning(e)
def test_tf_faster_rcnn(art_warning, get_mnist_dataset):
try:
master_seed(seed=1234, set_tensorflow=True)
# Only import if object detection module is available
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
# Define object detector
images = tf.placeholder(tf.float32, shape=[1, 28, 28, 1])
obj_dec = TensorFlowFasterRCNN(images=images)
# Get test data
(_, _), (x_test_mnist, y_test_mnist) = get_mnist_dataset
x_test_mnist = x_test_mnist[:1]
# First test predict
result = obj_dec.predict(x_test_mnist)
assert list(result[0].keys()) == ["boxes", "labels", "scores"]
assert result[0]["boxes"].shape == (300, 4)
expected_detection_boxes = np.asarray(
[0.008862, 0.003788, 0.070454, 0.175931])
np.testing.assert_array_almost_equal(result[0]["boxes"][2, :],
expected_detection_boxes,
decimal=3)
assert result[0]["scores"].shape == (300, )
expected_detection_scores = np.asarray([
2.196349e-04,
7.968055e-05,
7.811916e-05,
7.334248e-05,
6.868376e-05,
6.861838e-05,
6.756858e-05,
6.331169e-05,
6.313509e-05,
6.222352e-05,
])
np.testing.assert_array_almost_equal(result[0]["scores"][:10],
expected_detection_scores,
decimal=3)
assert result[0]["labels"].shape == (300, )
# expected_detection_classes = np.asarray([37, 15, 15, 66, 15, 15, 15, 63, 2, 66])
# np.testing.assert_array_almost_equal(result[0]["labels"][:10], expected_detection_classes)
# Then test loss gradient
# Create labels
y = [{
"boxes": result[0]["boxes"],
"labels": result[0]["labels"],
"scores": np.ones_like(result[0]["labels"])
}]
# Compute gradients
grads = obj_dec.loss_gradient(x_test_mnist[:1], y)
assert grads.shape == (1, 28, 28, 1)
expected_gradients = np.asarray([
[-0.00298723],
[-0.0039893],
[-0.00036253],
[0.01038542],
[0.01455704],
[0.00995643],
[0.00424966],
[0.00470569],
[0.00666382],
[0.0028694],
[0.00525351],
[0.00889174],
[0.0071413],
[0.00618231],
[0.00598106],
[0.0072665],
[0.00708815],
[0.00286943],
[0.00411595],
[0.00788978],
[0.00587319],
[0.00808631],
[0.01018151],
[0.00867905],
[0.00820272],
[0.00124911],
[-0.0042593],
[0.02380728],
])
np.testing.assert_array_almost_equal(grads[0, 0, :, :],
expected_gradients,
decimal=2)
# Then test loss gradient with standard format
# Create labels
result_tf = obj_dec.predict(x_test_mnist, standardise_output=False)
result = obj_dec.predict(x_test_mnist, standardise_output=True)
from art.estimators.object_detection.utils import convert_tf_to_pt
result_pt = convert_tf_to_pt(y=result_tf,
height=x_test_mnist.shape[1],
width=x_test_mnist.shape[2])
np.testing.assert_array_equal(result[0]["boxes"],
result_pt[0]["boxes"])
np.testing.assert_array_equal(result[0]["labels"],
result_pt[0]["labels"])
np.testing.assert_array_equal(result[0]["scores"],
result_pt[0]["scores"])
y = [{
"boxes": result[0]["boxes"],
"labels": result[0]["labels"],
"scores": np.ones_like(result[0]["labels"])
}]
# Compute gradients
grads = obj_dec.loss_gradient(x_test_mnist[:1],
y,
standardise_output=True)
assert grads.shape == (1, 28, 28, 1)
expected_gradients = np.asarray([
[-0.00095965],
[-0.00265362],
[-0.00031886],
[0.01132964],
[0.01674244],
[0.01262039],
[0.0063345],
[0.00673249],
[0.00618648],
[0.00422678],
[0.00542425],
[0.00814896],
[0.00919153],
[0.01068758],
[0.00929435],
[0.00877143],
[0.00747379],
[0.0050377],
[0.00656254],
[0.00799547],
[0.0051057],
[0.00714598],
[0.01090685],
[0.00787637],
[0.00709959],
[0.00047201],
[-0.00460457],
[0.02629307],
])
np.testing.assert_array_almost_equal(grads[0, 0, :, :],
expected_gradients,
decimal=2)
obj_dec._sess.close()
tf.reset_default_graph()
except ARTTestException as e:
art_warning(e)
def _test_image_as_input(self, sign_gradients):
# We must start a new graph
tf.reset_default_graph()
# Only import if object detection module is available
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
from art.attacks.evasion.shapeshifter import ShapeShifter
# Define object detector
images = tf.Variable(initial_value=np.zeros([1, 28, 28, 1]),
dtype=tf.float32)
obj_dec = TensorFlowFasterRCNN(images=images)
# Create labels
result = obj_dec.predict(self.x_test_mnist[:1].astype(np.float32))
groundtruth_boxes_list = [
result["detection_boxes"][i] for i in range(1)
]
groundtruth_classes_list = [
result["detection_classes"][i] for i in range(1)
]
groundtruth_weights_list = [
np.ones_like(r) for r in groundtruth_classes_list
]
y = {
"groundtruth_boxes_list": groundtruth_boxes_list,
"groundtruth_classes_list": groundtruth_classes_list,
"groundtruth_weights_list": groundtruth_weights_list,
}
# Define attack
attack = ShapeShifter(
estimator=obj_dec,
random_transform=lambda x: x + 1e-10,
box_classifier_weight=1.0,
box_localizer_weight=1.0,
rpn_classifier_weight=1.0,
rpn_localizer_weight=1.0,
box_iou_threshold=0.3,
box_victim_weight=1.0,
box_target_weight=1.0,
box_victim_cw_weight=1.0,
box_victim_cw_confidence=1.0,
box_target_cw_weight=1.0,
box_target_cw_confidence=1.0,
rpn_iou_threshold=0.3,
rpn_background_weight=1.0,
rpn_foreground_weight=1.0,
rpn_cw_weight=1.0,
rpn_cw_confidence=1.0,
similarity_weight=1.0,
learning_rate=0.1,
optimizer="RMSPropOptimizer",
momentum=0.01,
decay=0.01,
sign_gradients=sign_gradients,
random_size=2,
max_iter=2,
texture_as_input=False,
use_spectral=False,
soft_clip=True,
)
# Targeted attack
adv_x = attack.generate(x=self.x_test_mnist[:1].astype(np.float32),
label=y,
target_class=2,
victim_class=5)
self.assertTrue(adv_x.shape == (1, 28, 28, 1))
self.assertTrue((adv_x >= 0).all())
self.assertTrue((adv_x <= 1).all())
# Untargeted attack
adv_x = attack.generate(x=self.x_test_mnist[:1].astype(np.float32),
label=y,
victim_class=8)
self.assertTrue(adv_x.shape == (1, 28, 28, 1))
self.assertTrue((adv_x >= 0).all())
self.assertTrue((adv_x <= 1).all())
def _test_texture_as_input(self, sign_gradients, use_spectral, soft_clip):
# We must start a new graph
tf.reset_default_graph()
# Only import if object detection module is available
from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN
from art.attacks.evasion.shapeshifter import ShapeShifter
# Define object detector
images = tf.Variable(initial_value=np.zeros([1, 28, 28, 1]),
dtype=tf.float32)
obj_dec = TensorFlowFasterRCNN(images=images)
# Create labels
result = obj_dec.predict(self.x_test_mnist[:1].astype(np.float32))
groundtruth_boxes_list = [
result["detection_boxes"][i] for i in range(1)
]
groundtruth_classes_list = [
result["detection_classes"][i] for i in range(1)
]
groundtruth_weights_list = [
np.ones_like(r) for r in groundtruth_classes_list
]
y = {
"groundtruth_boxes_list": groundtruth_boxes_list,
"groundtruth_classes_list": groundtruth_classes_list,
"groundtruth_weights_list": groundtruth_weights_list,
}
# Define random transform
def random_transform(x):
background = np.random.rand(*x.shape)
image_frame = np.random.rand(*(list(x.shape[:-1]) + [4]))
y_ = y.copy()
y_["groundtruth_boxes_list"][
0] = y_["groundtruth_boxes_list"][0] + np.random.rand()
y_["groundtruth_weights_list"][
0] = y_["groundtruth_weights_list"][0] + np.random.rand()
return background, image_frame, y_
# Define attack
attack = ShapeShifter(
estimator=obj_dec,
random_transform=random_transform,
box_classifier_weight=1.0,
box_localizer_weight=1.0,
rpn_classifier_weight=1.0,
rpn_localizer_weight=1.0,
box_iou_threshold=0.3,
box_victim_weight=1.0,
box_target_weight=1.0,
box_victim_cw_weight=1.0,
box_victim_cw_confidence=1.0,
box_target_cw_weight=1.0,
box_target_cw_confidence=1.0,
rpn_iou_threshold=0.3,
rpn_background_weight=1.0,
rpn_foreground_weight=1.0,
rpn_cw_weight=1.0,
rpn_cw_confidence=1.0,
similarity_weight=1.0,
learning_rate=0.1,
optimizer="MomentumOptimizer",
momentum=0.01,
decay=0.01,
sign_gradients=sign_gradients,
random_size=2,
max_iter=2,
texture_as_input=True,
use_spectral=use_spectral,
soft_clip=soft_clip,
)
# Define rendering function
def rendering_function(background_phd, image_frame_phd,
current_texture):
current_image = background_phd + current_texture
current_image = tf.clip_by_value(current_image, 0, 1)
return current_image
# Targeted attack
adv_x = attack.generate(
x=self.x_test_mnist[:1].astype(np.float32),
label=y,
target_class=2,
victim_class=5,
rendering_function=rendering_function,
)
self.assertTrue(adv_x.shape == (1, 28, 28, 1))
# Untargeted attack
adv_x = attack.generate(
x=self.x_test_mnist[:1].astype(np.float32),
label=y,
target_class=8,
victim_class=8,
rendering_function=rendering_function,
)
self.assertTrue(adv_x.shape == (1, 28, 28, 1))
def test_check_params(self):
from art.estimators.object_detection import TensorFlowFasterRCNN
from art.attacks.evasion import ShapeShifter
images = tf.Variable(initial_value=np.zeros([1, 28, 28, 1]),
dtype=tf.float32)
obj_dec = TensorFlowFasterRCNN(images=images)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec, random_transform="1")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_classifier_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_classifier_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_localizer_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_localizer_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_classifier_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_classifier_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_localizer_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_localizer_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_iou_threshold=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_iou_threshold=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_cw_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_cw_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_cw_confidence=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_victim_cw_confidence=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_cw_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_cw_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_cw_confidence=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
box_target_cw_confidence=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_iou_threshold=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_iou_threshold=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_background_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_background_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_foreground_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_foreground_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_cw_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_cw_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_cw_confidence=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
rpn_cw_confidence=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
similarity_weight=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
similarity_weight=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
learning_rate=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
learning_rate=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="test")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="MomentumOptimizer",
momentum=1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="MomentumOptimizer",
momentum=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="RMSPropOptimizer",
momentum=0.5,
decay="1")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="RMSPropOptimizer",
momentum=0.5,
decay=-1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
optimizer="RMSPropOptimizer",
momentum=0.5,
decay=2.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
sign_gradients="true")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
random_size=1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
random_size=-1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
max_iter=1.0)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
max_iter=-1)
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
texture_as_input="true")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
use_spectral="true")
with self.assertRaises(ValueError):
_ = ShapeShifter(obj_dec,
random_transform=lambda x: x + 1e-10,
soft_clip="true")