def mnist_dataset(self):
dataset = torchvision.datasets.MNIST(
root=utils.python_file_dir(__file__) + '/data',
train=False,
download=True)
x = torch.unsqueeze((dataset.data / 255.0), 1)
y = dataset.targets
dataset = torch.utils.data.dataset.TensorDataset(x, y)
return dataset
def cifar10_dataset(self):
dataset = torchvision.datasets.CIFAR10(
root=utils.python_file_dir(__file__) + '/data',
train=False,
download=True)
x = torch.from_numpy(dataset.data / 255.0).permute(0, 3, 1, 2).float()
y = torch.tensor(dataset.targets)
dataset = torch.utils.data.dataset.TensorDataset(x, y)
return dataset
def test_cifar10_simple(self):
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.conv1 = torch.nn.Conv2d(3, 32, 3)
self.pool1 = torch.nn.MaxPool2d(2)
self.conv2 = torch.nn.Conv2d(32, 64, 3)
self.pool2 = torch.nn.MaxPool2d(2)
self.conv3 = torch.nn.Conv2d(64, 64, 3)
self.fc1 = torch.nn.Linear(64 * 4 * 4, 64)
self.fc2 = torch.nn.Linear(64, 10)
def forward(self, data):
data = torch.nn.functional.relu(self.conv1(data))
data = self.pool1(data)
data = torch.nn.functional.relu(self.conv2(data))
data = self.pool2(data)
data = torch.nn.functional.relu(self.conv3(data))
data = data.view(-1, 64 * 4 * 4)
data = torch.nn.functional.relu(self.fc1(data))
data = self.fc2(data)
return data
model = Model()
model.load_state_dict(
torch.load(
utils.python_file_dir(__file__) +
'/models/pytorch_cifar10_simple.pth'))
dataset = self.cifar10_dataset()
bounds = (0, 1)
num_classes = 10
measure_model = PyTorchModel(model)
accuracy = Accuracy()
measure_model.predict(dataset, [accuracy.update, accuracy.report])
neuron_coverage = NeuronCoverage()
measure_model.intermediate_layer_outputs(
dataset, [neuron_coverage.update, neuron_coverage.report])
robustness = Robustness(bounds)
measure_model.adversarial_samples(
dataset,
3,
bounds,
num_classes, [
robustness.update, robustness.report,
utils.draw_adversarial_samples
],
batch_size=1)
self.assertAlmostEqual(accuracy.get(1), 0.427500)
self.assertAlmostEqual(accuracy.get(5), 0.906400)
self.assertAlmostEqual(neuron_coverage.get(0.6), 0.534188, places=2)
self.assertAlmostEqual(robustness.success_rate, 1.000000)
def test_cifar10_simple(self):
model = mxnet.gluon.nn.Sequential()
with model.name_scope():
model.add(
mxnet.gluon.nn.Conv2D(channels=32,
kernel_size=3,
activation='relu'))
model.add(mxnet.gluon.nn.MaxPool2D(pool_size=2))
model.add(
mxnet.gluon.nn.Conv2D(channels=64,
kernel_size=3,
activation='relu'))
model.add(mxnet.gluon.nn.MaxPool2D(pool_size=2))
model.add(
mxnet.gluon.nn.Conv2D(channels=64,
kernel_size=3,
activation='relu'))
model.add(mxnet.gluon.nn.Flatten())
model.add(mxnet.gluon.nn.Dense(64, activation='relu'))
model.add(mxnet.gluon.nn.Dense(10))
model.load_parameters(
utils.python_file_dir(__file__) +
'/models/mxnet_cifar10_simple.params')
dataset = self.cifar10_dataset()
bounds = (0, 1)
num_classes = 10
measure_model = MXNetModel(model)
accuracy = Accuracy()
measure_model.predict(dataset, [accuracy.update, accuracy.report])
neuron_coverage = NeuronCoverage()
measure_model.intermediate_layer_outputs(
dataset, [neuron_coverage.update, neuron_coverage.report])
robustness = Robustness(bounds)
measure_model.adversarial_samples(
dataset,
3,
bounds,
num_classes, [
robustness.update, robustness.report,
utils.draw_adversarial_samples
],
batch_size=1)
self.assertAlmostEqual(accuracy.get(1), 0.687000)
self.assertAlmostEqual(accuracy.get(5), 0.966600)
self.assertAlmostEqual(neuron_coverage.get(0.6), 0.470085, places=2)
self.assertAlmostEqual(robustness.success_rate, 1.000000)
def __init__(self, batch_size=32, transform=None):
self._batch_size = batch_size
self._transform = transform
self._dir = utils.python_file_dir(__file__) + '/data/imagenet_val'
self._filenames = []
self._y = []
with open(self._dir + '/' + 'ILSVRC2012_validation_ground_truth.txt', 'r') as f:
lines = f.readlines()
for line in lines:
splits = line.split('---')
if len(splits) != 5:
continue
self._filenames.append(splits[0])
self._y.append(int(splits[2]))
self._y = np.array(self._y, dtype=int)
def test_mnist_simple(self):
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.fc1 = torch.nn.Linear(1 * 28 * 28, 128)
self.fc2 = torch.nn.Linear(128, 64)
self.fc3 = torch.nn.Linear(64, 10)
def forward(self, data):
data = data.view(-1, 1 * 28 * 28)
data = torch.nn.functional.relu(self.fc1(data))
data = torch.nn.functional.relu(self.fc2(data))
data = self.fc3(data)
return data
model = Model()
model.load_state_dict(
torch.load(
utils.python_file_dir(__file__) +
'/models/pytorch_mnist_simple.pth'))
dataset = self.mnist_dataset()
bounds = (0, 1)
num_classes = 10
measure_model = PyTorchModel(model)
accuracy = Accuracy()
measure_model.predict(dataset, [accuracy.update, accuracy.report])
neuron_coverage = NeuronCoverage()
measure_model.intermediate_layer_outputs(
dataset, [neuron_coverage.update, neuron_coverage.report])
robustness = Robustness(bounds)
measure_model.adversarial_samples(
dataset,
3,
bounds,
num_classes, [
robustness.update, robustness.report,
utils.draw_adversarial_samples
],
batch_size=1)
self.assertAlmostEqual(accuracy.get(1), 0.962200)
self.assertAlmostEqual(accuracy.get(5), 0.998900)
self.assertAlmostEqual(neuron_coverage.get(0.7), 0.876238, places=2)
self.assertAlmostEqual(robustness.success_rate, 1.000000)
def __init__(self, preprocess):
self._preprocess = preprocess
self._dir = utils.python_file_dir(__file__) + '/data/imagenet_val'
self._filenames = []
self._y = []
with open(
self._dir + '/' + 'ILSVRC2012_validation_ground_truth.txt',
'r') as f:
lines = f.readlines()
for line in lines:
splits = line.split('---')
if len(splits) != 5:
continue
self._filenames.append(splits[0])
self._y.append(int(splits[2]))
self._y = mxnet.nd.array(self._y, dtype=int)
def test_mnist_simple(self):
keras.backend.set_learning_phase(0)
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
model = keras.models.load_model(utils.python_file_dir(__file__) + '/models/keras_mnist_simple.h5')
model.trainable = False
x, y = self.mnist_data()
bounds = (0, 1)
measure_model = KerasModel(model)
accuracy = Accuracy()
measure_model.predict(x, y, [accuracy.update, accuracy.report])
neuron_coverage = NeuronCoverage()
measure_model.intermediate_layer_outputs(x, [neuron_coverage.update, neuron_coverage.report])
robustness = Robustness(bounds)
measure_model.adversarial_samples(x[:5], y[:5], 3, bounds, [robustness.update, robustness.report, utils.draw_adversarial_samples])
self.assertAlmostEqual(accuracy.get(1), 0.991000)
self.assertAlmostEqual(accuracy.get(5), 1.000000)
self.assertAlmostEqual(neuron_coverage.get(0.6), 0.589744, places=2)
self.assertAlmostEqual(robustness.success_rate, 0.6666666667)
