def main(args):
goal = args.goal
distance = args.distance
method = args.method
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
SESSION = tf.Session(config=config)
JPEG_SESSION = tf.Session(config=config)
paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION)
IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3)
MODEL = IncV3_Jpeg()
ITERATION = 100
LABEL_OFFSET = MODEL.n_class - 1000
if distance == 'l_inf':
# L_inf parameters
MAGNITUDE = 16.0 / 255.0
ALPHA = 2.0 / 255.0
SPSA_LR = ALPHA
else:
# L_2 parameters
MAGNITUDE = np.sqrt(1e-3 * MODEL.x_shape[0] * MODEL.x_shape[1] * MODEL.x_shape[2])
ALPHA = MAGNITUDE * 0.15
SPSA_LR = 0.01
CONFIGS = {
'deepfool': {},
'nes': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'learning_rate': ALPHA,
'min_lr': ALPHA / 10,
'lr_tuning': True,
'plateau_length': 20,
},
'spsa': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'learning_rate': SPSA_LR,
},
'nattack': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 0.1,
'learning_rate': 0.02,
},
'boundary': {
'iteration': 10000,
'max_directions': 25,
'max_queries': 20000,
'spherical_step': 1e-2,
'source_step': 1e-2,
'step_adaptation': 1.5,
'logging': False,
},
'evolutionary': {
'max_queries': 20000,
'mu': 1e-2,
'sigma': 3e-2,
'sample_size': 32,
'logging': False,
},
}
ATTACKS = {
'deepfool': DeepFool,
'nes': NES,
'spsa': SPSA,
'nattack': NAttack,
'boundary': Boundary,
'evolutionary': Evolutionary,
}
MODEL.load(session=SESSION)
XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape))
YS_PH = tf.placeholder(MODEL.y_dtype, (None,))
LOGITS, LABELS = MODEL.logits_and_labels(XS_PH)
def distance_l_2(a, b):
d = np.reshape(a - b, (a.shape[0], -1))
return np.sqrt((d ** 2).sum(axis=1))
def distance_l_inf(a, b):
d = np.reshape(a - b, (a.shape[0], -1))
return np.abs(d).max(axis=1)
class IterationBenchmark(object):
def __init__(self, method, basic_configs):
RUNS = {
'deepfool': self._run_deepfool,
'nes': self._run_score_based,
'spsa': self._run_score_based,
'nattack': self._run_score_based,
'boundary': self._run_decision_based,
'evolutionary': self._run_decision_based,
}
self.method = method
self.basic_configs = basic_configs
self.config = CONFIGS[self.method]
self.attack = ATTACKS[self.method](MODEL, 1)
self.goal = basic_configs['goal']
if basic_configs['distance_metric'] == 'l_2':
self._distance = distance_l_2
else:
self._distance = distance_l_inf
self._run = RUNS[self.method]
def _run_deepfool(self, xs, ys, ys_target):
self.attack.config(
iteration=ITERATION,
**self.basic_configs,
**self.config
)
rs = dict()
for i, xs_adv in enumerate(self.attack.batch_attack_iterator(
xs, ys, ys_target, SESSION)):
print(" {}".format(i + 1))
lb = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0]
di = self._distance(xs_adv, xs)[0]
rs[i + 1] = (lb, di)
return rs, xs_adv
def _run_score_based(self, xs, ys, ys_target):
self.attack.config(
magnitude=MAGNITUDE,
**self.basic_configs,
**self.config
)
xs_adv, q = self.attack.batch_attack(xs, ys, ys_target, SESSION)
print(ys, ys_target, SESSION.run(LABELS, feed_dict={XS_PH: xs})[
0], SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0])
print(np.max(xs_adv), np.min(xs_adv), np.max(
xs_adv - xs), np.min(xs_adv - xs), q)
lb = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0]
return (lb, q)
def _run_decision_based(self, xs, ys, ys_target):
if self.goal == 'ut' or self.goal == 'tm':
starting_point = np.random.uniform(
MODEL.x_min, MODEL.x_max, size=MODEL.x_shape).astype(
MODEL.x_dtype.as_numpy_dtype)
while SESSION.run(LABELS, feed_dict={XS_PH: starting_point[np.newaxis]}) == ys:
starting_point = np.random.uniform(
MODEL.x_min, MODEL.x_max, size=MODEL.x_shape).astype(
MODEL.x_dtype.as_numpy_dtype)
else:
starting_point = xs_for_each_label[ys_target].squeeze(axis=0)
self.attack.config(
**self.basic_configs,
**self.config)
xs_adv, dis_per_query = self.attack.batch_attack(
xs, ys, ys_target, SESSION, starting_point)
print(ys, ys_target, SESSION.run(LABELS, feed_dict={XS_PH: xs})[
0], SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0])
print(np.linalg.norm(xs_adv - xs), dis_per_query[-1])
return (xs_adv, dis_per_query)
def run(self, start, end):
rs = []
for filenames, xs, ys, ys_target in load_batches_imagenet_test(batch_size=1,
x_min=MODEL.x_min,
x_max=MODEL.x_max,
x_shape=MODEL.x_shape,
x_dtype=MODEL.x_dtype,
y_dtype=MODEL.y_dtype,
start=start,
end=end,
label_offset=LABEL_OFFSET,
return_target_class=True):
print(filenames)
rs.append(self._run(xs, ys, ys_target))
return rs
def prepare_xs_for_each_label():
n_class = MODEL.n_class
xs_for_each_label = np.zeros([n_class, *MODEL.x_shape]).astype(
MODEL.x_dtype.as_numpy_dtype)
for i in range(n_class):
for xs in load_image_of_class(label=i,
x_min=MODEL.x_min,
x_max=MODEL.x_max,
x_shape=MODEL.x_shape,
x_dtype=MODEL.x_dtype,
y_dtype=MODEL.y_dtype,
label_offset=LABEL_OFFSET):
lb = SESSION.run(LABELS, feed_dict={XS_PH: xs})[0]
if lb == i:
xs_for_each_label[i] = xs
break
return xs_for_each_label
output = args.output
basic_configs = {
'goal': goal,
'distance_metric': distance
}
if method == 'boundary' or method == 'evolutionary':
xs_for_each_label = prepare_xs_for_each_label()
benchmark = IterationBenchmark(method, basic_configs)
rs = benchmark.run(args.start, args.end)
np.save(output, rs)
from realsafe.cifar10.ResNet56 import ResNet56
from realsafe.benchmark.distortion_benchmark import DistortionBenchmarkBuilder
from realsafe.benchmark.utils import distortion_benchmark_parser, new_session
from realsafe.defense.jpeg import jpeg_classifier_differentiable
import tensorflow as tf
import numpy as np
SESSION = tf.Session()
args = distortion_benchmark_parser()
JPEG_SESSION = new_session()
paras = (0.0, 1.0, (32, 32, 3), tf.float32, tf.int32, 10, 75, JPEG_SESSION)
ResNet56_Jpeg = jpeg_classifier_differentiable(*paras)(ResNet56)
MODEL = ResNet56_Jpeg()
MODEL.load(session=SESSION)
ITERATION = 20
builder = DistortionBenchmarkBuilder()
builder.init_distortion_l_2(32.0 * (MODEL.x_max - MODEL.x_min))
builder.init_distortion_l_inf(1.0 * (MODEL.x_max - MODEL.x_min))
builder.search_steps(0)
builder.binsearch_steps(14)
builder.batch_size(100)
builder.config_init_l_2('bim', {})
builder.config_l_2('bim', {
'iteration': ITERATION,
'session': SESSION
def main(args):
goal = args.goal
distance = args.distance
method = args.method
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
SESSION = tf.Session(config=config)
JPEG_SESSION = tf.Session(config=config)
paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION)
IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3)
MODEL = IncV3_Jpeg()
BIN_SEARCH_STEPS = 10
LABEL_OFFSET = MODEL.n_class - 1000
if distance == 'l_inf':
# L_inf parameters
INIT_DISTORTION = 0.1 * (MODEL.x_max - MODEL.x_min)
SPSA_LR_MUL = 0.15
else:
# L_2 parameters
INIT_DISTORTION = np.sqrt(1e-3 * MODEL.x_shape[0] * MODEL.x_shape[1] * MODEL.x_shape[2]) * (MODEL.x_max - MODEL.x_min)
SPSA_LR_MUL = 0.01 / INIT_DISTORTION
CONFIGS = {
'nes': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'lr_tuning': True,
'plateau_length': 20,
'return_details': False,
},
'spsa': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'return_details': False,
},
'nattack': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 0.1,
'learning_rate': 0.02,
'return_details': False,
},
}
ATTACKS = {
'nes': NES,
'spsa': SPSA,
'nattack': NAttack,
}
MODEL.load(session=SESSION)
XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape))
YS_PH = tf.placeholder(MODEL.y_dtype, (None,))
LOGITS, LABELS = MODEL.logits_and_labels(XS_PH)
class SuccessRateBenchmark(object):
def __init__(self, method, basic_configs):
RUNS = {
'nes': SuccessRateBenchmark._bin_search_nes,
'spsa': SuccessRateBenchmark._bin_search_nes,
'nattack': SuccessRateBenchmark._bin_search_nes,
}
self.config = CONFIGS[method]
self.basic_configs = basic_configs
self.goal = basic_configs['goal']
self.method = method
self.attack = ATTACKS[method](MODEL, 1)
self._run = RUNS[method]
self._optimized_configured = False
def _bin_search_nes(self, xs, ys, ys_target):
found = np.array([False])
hi = np.zeros(shape=1, dtype=np.float32) + INIT_DISTORTION
lo = np.zeros(shape=1, dtype=np.float32)
xs_adv = np.zeros_like(xs)
for i in range(10):
print(i, hi, lo)
if self.method == 'nes':
specific_config = {'magnitude': hi,
'learning_rate': hi * 0.15,
'min_lr': hi * 0.015
}
elif self.method == 'spsa':
specific_config = {'magnitude': hi,
'learning_rate': hi * SPSA_LR_MUL}
elif self.method == 'nattack':
specific_config = {'magnitude': hi}
self.attack.config(
**specific_config, **self.basic_configs, **self.config
)
xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION)
ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_})
flag = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys
cond = np.logical_and(np.logical_not(found), flag)
xs_adv[cond] = xs_adv_[cond]
found[cond] = True
lo[np.logical_not(found)] = hi[np.logical_not(found)]
hi[np.logical_not(found)] *= 2.0
if found.all():
break
else:
return xs_adv
for i in range(BIN_SEARCH_STEPS):
mi = (lo + hi) / 2
print(i, mi)
if self.method == 'nes':
specific_config = {'magnitude': mi,
'learning_rate': mi * 0.15,
'min_lr': mi * 0.015}
elif self.method == 'spsa':
specific_config = {'magnitude': mi,
'learning_rate': mi * SPSA_LR_MUL}
elif self.method == 'nattack':
specific_config = {'magnitude': mi}
self.attack.config(
**specific_config, **self.basic_configs, **self.config
)
xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION)
ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_})
succ = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys
not_succ = np.logical_not(succ)
hi[succ] = mi[succ]
lo[not_succ] = mi[not_succ]
xs_adv[succ] = xs_adv_[succ]
return xs_adv
def run(self, start, end):
xs_adv = np.zeros((end - start,) + MODEL.x_shape)
idx = 0
for filenames, xs, ys, ys_target in load_batches_imagenet_test(batch_size=1,
x_min=MODEL.x_min,
x_max=MODEL.x_max,
x_shape=MODEL.x_shape,
x_dtype=MODEL.x_dtype,
y_dtype=MODEL.y_dtype,
start=start,
end=end,
label_offset=LABEL_OFFSET,
return_target_class=True):
print(filenames)
xs_adv[idx] = self._run(self, xs, ys, ys_target)
idx += 1
return xs_adv
output = args.output
basic_configs = {
'goal': goal,
'distance_metric': distance
}
benchmark = SuccessRateBenchmark(method, basic_configs)
xs_adv = benchmark.run(args.start, args.end)
np.save(output, xs_adv)
from realsafe.dataset.ImageNet import load_batches_imagenet_test
from realsafe.benchmark.utils import imagenet_iteration_benchmark_parser
from realsafe.benchmark.utils import new_session
from realsafe.ImageNet.inception_v3 import Inception_V3
from realsafe.defense.jpeg import jpeg_classifier_differentiable
from realsafe.benchmark.iteration_benchmark import IterationBenchmarkBuilder
import tensorflow as tf
import numpy as np
import os
BATCH_SIZE = 50
JPEG_SESSION = new_session()
SESSION = tf.Session()
paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION)
IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3)
args = imagenet_iteration_benchmark_parser()
MODEL = IncV3_Jpeg()
print('x_min={}\nx_max={}\nx_shape={}\nn_class={}'.format(
MODEL.x_min, MODEL.x_max, MODEL.x_shape, MODEL.n_class))
LABEL_OFFSET = MODEL.n_class - 1000
ITERATION = 100
MAGNITUDE_L_INF = 16.0 / 255.0
ALPHA_L_INF = 2.0 / 255.0
SPSA_LR_L_INF = ALPHA_L_INF
MAGNITUDE_L_2 = np.sqrt(1e-3 * np.prod(MODEL.x_shape))
def main(args):
goal = args.goal
distance = args.distance
method = args.method
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
SESSION = tf.Session(config=config)
JPEG_SESSION = tf.Session(config=config)
paras = (0.0, 1.0, (32, 32, 3), tf.float32, tf.int32, 10, 75, JPEG_SESSION)
ResNet56_Jpeg = jpeg_classifier_differentiable(*paras)(ResNet56)
MODEL = ResNet56_Jpeg()
BIN_SEARCH_STEPS = 10
if distance == 'l_inf':
# L_inf parameters
INIT_DISTORTION = 0.1 * (MODEL.x_max - MODEL.x_min)
SPSA_LR_MUL = 0.15
else:
# L_2 parameters
INIT_DISTORTION = 1.0 * (MODEL.x_max - MODEL.x_min)
SPSA_LR_MUL = 0.005
CONFIGS = {
'nes': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'lr_tuning': True,
'plateau_length': 20,
'return_details': False,
},
'spsa': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min),
'return_details': False,
},
'nattack': {
'max_queries': 20000,
'samples_per_draw': 100,
'sigma': 0.1,
'learning_rate': 0.02,
'return_details': False,
},
}
ATTACKS = {
'nes': NES,
'spsa': SPSA,
'nattack': NAttack,
}
MODEL.load(session=SESSION)
XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape))
YS_PH = tf.placeholder(MODEL.y_dtype, (None, ))
LOGITS, LABELS = MODEL.logits_and_labels(XS_PH)
class SuccessRateBenchmark(object):
def __init__(self, method, basic_configs):
RUNS = {
'nes': SuccessRateBenchmark._bin_search_nes,
'spsa': SuccessRateBenchmark._bin_search_nes,
'nattack': SuccessRateBenchmark._bin_search_nes,
}
self.config = CONFIGS[method]
self.basic_configs = basic_configs
self.goal = basic_configs['goal']
self.method = method
self.attack = ATTACKS[method](MODEL, 1)
self._run = RUNS[method]
self._optimized_configured = False
def run(self, xs, ys, ys_target):
xs_adv = np.zeros_like(xs)
for lo in range(len(xs)):
print(lo)
s = slice(lo, lo + 1)
xs_adv[s] = self._run(self, xs[s], ys[s], ys_target[s])
return xs_adv
def _bin_search_nes(self, xs, ys, ys_target):
found = np.array([False])
hi = np.zeros(shape=1, dtype=np.float32) + INIT_DISTORTION
lo = np.zeros(shape=1, dtype=np.float32)
xs_adv = np.zeros_like(xs)
for i in range(10):
print(i, hi, lo)
if self.method == 'nes':
specific_config = {
'magnitude': hi,
'learning_rate': hi * 0.15,
'min_lr': hi * 0.015
}
elif self.method == 'spsa':
specific_config = {
'magnitude': hi,
'learning_rate': hi * SPSA_LR_MUL
}
elif self.method == 'nattack':
specific_config = {'magnitude': hi}
self.attack.config(**specific_config, **self.basic_configs,
**self.config)
xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION)
ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_})
flag = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys
cond = np.logical_and(np.logical_not(found), flag)
xs_adv[cond] = xs_adv_[cond]
found[cond] = True
lo[np.logical_not(found)] = hi[np.logical_not(found)]
hi[np.logical_not(found)] *= 2.0
if found.all():
break
for i in range(BIN_SEARCH_STEPS):
mi = (lo + hi) / 2
print(i, mi)
if self.method == 'nes':
specific_config = {
'magnitude': mi,
'learning_rate': mi * 0.15,
'min_lr': mi * 0.015
}
elif self.method == 'spsa':
specific_config = {
'magnitude': mi,
'learning_rate': mi * SPSA_LR_MUL
}
elif self.method == 'nattack':
specific_config = {'magnitude': mi}
self.attack.config(**specific_config, **self.basic_configs,
**self.config)
xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION)
ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_})
succ = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys
not_succ = np.logical_not(succ)
hi[succ] = mi[succ]
lo[not_succ] = mi[not_succ]
xs_adv[succ] = xs_adv_[succ]
return xs_adv
xs = np.load(args.xs).astype(MODEL.x_dtype.as_numpy_dtype)
xs = (xs / 255.0) * (MODEL.x_max - MODEL.x_min) + MODEL.x_min
ys = np.load(args.ys).astype(MODEL.y_dtype.as_numpy_dtype)
ys_target = np.load(args.ys_target).astype(MODEL.y_dtype.as_numpy_dtype)
output = args.output
basic_configs = {'goal': goal, 'distance_metric': distance}
benchmark = SuccessRateBenchmark(method, basic_configs)
xs_adv = benchmark.run(xs, ys, ys_target)
xs_adv = (xs_adv - MODEL.x_min) / (MODEL.x_max - MODEL.x_min) * 255.0
np.save(output, xs_adv)