def main(args): goal = args.goal distance = args.distance method = args.method config = tf.ConfigProto() config.gpu_options.allow_growth = True SESSION = tf.Session(config=config) JPEG_SESSION = tf.Session(config=config) paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION) IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3) MODEL = IncV3_Jpeg() ITERATION = 100 LABEL_OFFSET = MODEL.n_class - 1000 if distance == 'l_inf': # L_inf parameters MAGNITUDE = 16.0 / 255.0 ALPHA = 2.0 / 255.0 SPSA_LR = ALPHA else: # L_2 parameters MAGNITUDE = np.sqrt(1e-3 * MODEL.x_shape[0] * MODEL.x_shape[1] * MODEL.x_shape[2]) ALPHA = MAGNITUDE * 0.15 SPSA_LR = 0.01 CONFIGS = { 'deepfool': {}, 'nes': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'learning_rate': ALPHA, 'min_lr': ALPHA / 10, 'lr_tuning': True, 'plateau_length': 20, }, 'spsa': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'learning_rate': SPSA_LR, }, 'nattack': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 0.1, 'learning_rate': 0.02, }, 'boundary': { 'iteration': 10000, 'max_directions': 25, 'max_queries': 20000, 'spherical_step': 1e-2, 'source_step': 1e-2, 'step_adaptation': 1.5, 'logging': False, }, 'evolutionary': { 'max_queries': 20000, 'mu': 1e-2, 'sigma': 3e-2, 'sample_size': 32, 'logging': False, }, } ATTACKS = { 'deepfool': DeepFool, 'nes': NES, 'spsa': SPSA, 'nattack': NAttack, 'boundary': Boundary, 'evolutionary': Evolutionary, } MODEL.load(session=SESSION) XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape)) YS_PH = tf.placeholder(MODEL.y_dtype, (None,)) LOGITS, LABELS = MODEL.logits_and_labels(XS_PH) def distance_l_2(a, b): d = np.reshape(a - b, (a.shape[0], -1)) return np.sqrt((d ** 2).sum(axis=1)) def distance_l_inf(a, b): d = np.reshape(a - b, (a.shape[0], -1)) return np.abs(d).max(axis=1) class IterationBenchmark(object): def __init__(self, method, basic_configs): RUNS = { 'deepfool': self._run_deepfool, 'nes': self._run_score_based, 'spsa': self._run_score_based, 'nattack': self._run_score_based, 'boundary': self._run_decision_based, 'evolutionary': self._run_decision_based, } self.method = method self.basic_configs = basic_configs self.config = CONFIGS[self.method] self.attack = ATTACKS[self.method](MODEL, 1) self.goal = basic_configs['goal'] if basic_configs['distance_metric'] == 'l_2': self._distance = distance_l_2 else: self._distance = distance_l_inf self._run = RUNS[self.method] def _run_deepfool(self, xs, ys, ys_target): self.attack.config( iteration=ITERATION, **self.basic_configs, **self.config ) rs = dict() for i, xs_adv in enumerate(self.attack.batch_attack_iterator( xs, ys, ys_target, SESSION)): print(" {}".format(i + 1)) lb = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0] di = self._distance(xs_adv, xs)[0] rs[i + 1] = (lb, di) return rs, xs_adv def _run_score_based(self, xs, ys, ys_target): self.attack.config( magnitude=MAGNITUDE, **self.basic_configs, **self.config ) xs_adv, q = self.attack.batch_attack(xs, ys, ys_target, SESSION) print(ys, ys_target, SESSION.run(LABELS, feed_dict={XS_PH: xs})[ 0], SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0]) print(np.max(xs_adv), np.min(xs_adv), np.max( xs_adv - xs), np.min(xs_adv - xs), q) lb = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0] return (lb, q) def _run_decision_based(self, xs, ys, ys_target): if self.goal == 'ut' or self.goal == 'tm': starting_point = np.random.uniform( MODEL.x_min, MODEL.x_max, size=MODEL.x_shape).astype( MODEL.x_dtype.as_numpy_dtype) while SESSION.run(LABELS, feed_dict={XS_PH: starting_point[np.newaxis]}) == ys: starting_point = np.random.uniform( MODEL.x_min, MODEL.x_max, size=MODEL.x_shape).astype( MODEL.x_dtype.as_numpy_dtype) else: starting_point = xs_for_each_label[ys_target].squeeze(axis=0) self.attack.config( **self.basic_configs, **self.config) xs_adv, dis_per_query = self.attack.batch_attack( xs, ys, ys_target, SESSION, starting_point) print(ys, ys_target, SESSION.run(LABELS, feed_dict={XS_PH: xs})[ 0], SESSION.run(LABELS, feed_dict={XS_PH: xs_adv})[0]) print(np.linalg.norm(xs_adv - xs), dis_per_query[-1]) return (xs_adv, dis_per_query) def run(self, start, end): rs = [] for filenames, xs, ys, ys_target in load_batches_imagenet_test(batch_size=1, x_min=MODEL.x_min, x_max=MODEL.x_max, x_shape=MODEL.x_shape, x_dtype=MODEL.x_dtype, y_dtype=MODEL.y_dtype, start=start, end=end, label_offset=LABEL_OFFSET, return_target_class=True): print(filenames) rs.append(self._run(xs, ys, ys_target)) return rs def prepare_xs_for_each_label(): n_class = MODEL.n_class xs_for_each_label = np.zeros([n_class, *MODEL.x_shape]).astype( MODEL.x_dtype.as_numpy_dtype) for i in range(n_class): for xs in load_image_of_class(label=i, x_min=MODEL.x_min, x_max=MODEL.x_max, x_shape=MODEL.x_shape, x_dtype=MODEL.x_dtype, y_dtype=MODEL.y_dtype, label_offset=LABEL_OFFSET): lb = SESSION.run(LABELS, feed_dict={XS_PH: xs})[0] if lb == i: xs_for_each_label[i] = xs break return xs_for_each_label output = args.output basic_configs = { 'goal': goal, 'distance_metric': distance } if method == 'boundary' or method == 'evolutionary': xs_for_each_label = prepare_xs_for_each_label() benchmark = IterationBenchmark(method, basic_configs) rs = benchmark.run(args.start, args.end) np.save(output, rs)
from realsafe.cifar10.ResNet56 import ResNet56 from realsafe.benchmark.distortion_benchmark import DistortionBenchmarkBuilder from realsafe.benchmark.utils import distortion_benchmark_parser, new_session from realsafe.defense.jpeg import jpeg_classifier_differentiable import tensorflow as tf import numpy as np SESSION = tf.Session() args = distortion_benchmark_parser() JPEG_SESSION = new_session() paras = (0.0, 1.0, (32, 32, 3), tf.float32, tf.int32, 10, 75, JPEG_SESSION) ResNet56_Jpeg = jpeg_classifier_differentiable(*paras)(ResNet56) MODEL = ResNet56_Jpeg() MODEL.load(session=SESSION) ITERATION = 20 builder = DistortionBenchmarkBuilder() builder.init_distortion_l_2(32.0 * (MODEL.x_max - MODEL.x_min)) builder.init_distortion_l_inf(1.0 * (MODEL.x_max - MODEL.x_min)) builder.search_steps(0) builder.binsearch_steps(14) builder.batch_size(100) builder.config_init_l_2('bim', {}) builder.config_l_2('bim', { 'iteration': ITERATION, 'session': SESSION
def main(args): goal = args.goal distance = args.distance method = args.method config = tf.ConfigProto() config.gpu_options.allow_growth = True SESSION = tf.Session(config=config) JPEG_SESSION = tf.Session(config=config) paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION) IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3) MODEL = IncV3_Jpeg() BIN_SEARCH_STEPS = 10 LABEL_OFFSET = MODEL.n_class - 1000 if distance == 'l_inf': # L_inf parameters INIT_DISTORTION = 0.1 * (MODEL.x_max - MODEL.x_min) SPSA_LR_MUL = 0.15 else: # L_2 parameters INIT_DISTORTION = np.sqrt(1e-3 * MODEL.x_shape[0] * MODEL.x_shape[1] * MODEL.x_shape[2]) * (MODEL.x_max - MODEL.x_min) SPSA_LR_MUL = 0.01 / INIT_DISTORTION CONFIGS = { 'nes': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'lr_tuning': True, 'plateau_length': 20, 'return_details': False, }, 'spsa': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'return_details': False, }, 'nattack': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 0.1, 'learning_rate': 0.02, 'return_details': False, }, } ATTACKS = { 'nes': NES, 'spsa': SPSA, 'nattack': NAttack, } MODEL.load(session=SESSION) XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape)) YS_PH = tf.placeholder(MODEL.y_dtype, (None,)) LOGITS, LABELS = MODEL.logits_and_labels(XS_PH) class SuccessRateBenchmark(object): def __init__(self, method, basic_configs): RUNS = { 'nes': SuccessRateBenchmark._bin_search_nes, 'spsa': SuccessRateBenchmark._bin_search_nes, 'nattack': SuccessRateBenchmark._bin_search_nes, } self.config = CONFIGS[method] self.basic_configs = basic_configs self.goal = basic_configs['goal'] self.method = method self.attack = ATTACKS[method](MODEL, 1) self._run = RUNS[method] self._optimized_configured = False def _bin_search_nes(self, xs, ys, ys_target): found = np.array([False]) hi = np.zeros(shape=1, dtype=np.float32) + INIT_DISTORTION lo = np.zeros(shape=1, dtype=np.float32) xs_adv = np.zeros_like(xs) for i in range(10): print(i, hi, lo) if self.method == 'nes': specific_config = {'magnitude': hi, 'learning_rate': hi * 0.15, 'min_lr': hi * 0.015 } elif self.method == 'spsa': specific_config = {'magnitude': hi, 'learning_rate': hi * SPSA_LR_MUL} elif self.method == 'nattack': specific_config = {'magnitude': hi} self.attack.config( **specific_config, **self.basic_configs, **self.config ) xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION) ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_}) flag = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys cond = np.logical_and(np.logical_not(found), flag) xs_adv[cond] = xs_adv_[cond] found[cond] = True lo[np.logical_not(found)] = hi[np.logical_not(found)] hi[np.logical_not(found)] *= 2.0 if found.all(): break else: return xs_adv for i in range(BIN_SEARCH_STEPS): mi = (lo + hi) / 2 print(i, mi) if self.method == 'nes': specific_config = {'magnitude': mi, 'learning_rate': mi * 0.15, 'min_lr': mi * 0.015} elif self.method == 'spsa': specific_config = {'magnitude': mi, 'learning_rate': mi * SPSA_LR_MUL} elif self.method == 'nattack': specific_config = {'magnitude': mi} self.attack.config( **specific_config, **self.basic_configs, **self.config ) xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION) ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_}) succ = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys not_succ = np.logical_not(succ) hi[succ] = mi[succ] lo[not_succ] = mi[not_succ] xs_adv[succ] = xs_adv_[succ] return xs_adv def run(self, start, end): xs_adv = np.zeros((end - start,) + MODEL.x_shape) idx = 0 for filenames, xs, ys, ys_target in load_batches_imagenet_test(batch_size=1, x_min=MODEL.x_min, x_max=MODEL.x_max, x_shape=MODEL.x_shape, x_dtype=MODEL.x_dtype, y_dtype=MODEL.y_dtype, start=start, end=end, label_offset=LABEL_OFFSET, return_target_class=True): print(filenames) xs_adv[idx] = self._run(self, xs, ys, ys_target) idx += 1 return xs_adv output = args.output basic_configs = { 'goal': goal, 'distance_metric': distance } benchmark = SuccessRateBenchmark(method, basic_configs) xs_adv = benchmark.run(args.start, args.end) np.save(output, xs_adv)
from realsafe.dataset.ImageNet import load_batches_imagenet_test from realsafe.benchmark.utils import imagenet_iteration_benchmark_parser from realsafe.benchmark.utils import new_session from realsafe.ImageNet.inception_v3 import Inception_V3 from realsafe.defense.jpeg import jpeg_classifier_differentiable from realsafe.benchmark.iteration_benchmark import IterationBenchmarkBuilder import tensorflow as tf import numpy as np import os BATCH_SIZE = 50 JPEG_SESSION = new_session() SESSION = tf.Session() paras = (0.0, 1.0, (299, 299, 3), tf.float32, tf.int32, 1001, 75, JPEG_SESSION) IncV3_Jpeg = jpeg_classifier_differentiable(*paras)(Inception_V3) args = imagenet_iteration_benchmark_parser() MODEL = IncV3_Jpeg() print('x_min={}\nx_max={}\nx_shape={}\nn_class={}'.format( MODEL.x_min, MODEL.x_max, MODEL.x_shape, MODEL.n_class)) LABEL_OFFSET = MODEL.n_class - 1000 ITERATION = 100 MAGNITUDE_L_INF = 16.0 / 255.0 ALPHA_L_INF = 2.0 / 255.0 SPSA_LR_L_INF = ALPHA_L_INF MAGNITUDE_L_2 = np.sqrt(1e-3 * np.prod(MODEL.x_shape))
def main(args): goal = args.goal distance = args.distance method = args.method config = tf.ConfigProto() config.gpu_options.allow_growth = True SESSION = tf.Session(config=config) JPEG_SESSION = tf.Session(config=config) paras = (0.0, 1.0, (32, 32, 3), tf.float32, tf.int32, 10, 75, JPEG_SESSION) ResNet56_Jpeg = jpeg_classifier_differentiable(*paras)(ResNet56) MODEL = ResNet56_Jpeg() BIN_SEARCH_STEPS = 10 if distance == 'l_inf': # L_inf parameters INIT_DISTORTION = 0.1 * (MODEL.x_max - MODEL.x_min) SPSA_LR_MUL = 0.15 else: # L_2 parameters INIT_DISTORTION = 1.0 * (MODEL.x_max - MODEL.x_min) SPSA_LR_MUL = 0.005 CONFIGS = { 'nes': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'lr_tuning': True, 'plateau_length': 20, 'return_details': False, }, 'spsa': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 1e-3 * (MODEL.x_max - MODEL.x_min), 'return_details': False, }, 'nattack': { 'max_queries': 20000, 'samples_per_draw': 100, 'sigma': 0.1, 'learning_rate': 0.02, 'return_details': False, }, } ATTACKS = { 'nes': NES, 'spsa': SPSA, 'nattack': NAttack, } MODEL.load(session=SESSION) XS_PH = tf.placeholder(MODEL.x_dtype, (None, *MODEL.x_shape)) YS_PH = tf.placeholder(MODEL.y_dtype, (None, )) LOGITS, LABELS = MODEL.logits_and_labels(XS_PH) class SuccessRateBenchmark(object): def __init__(self, method, basic_configs): RUNS = { 'nes': SuccessRateBenchmark._bin_search_nes, 'spsa': SuccessRateBenchmark._bin_search_nes, 'nattack': SuccessRateBenchmark._bin_search_nes, } self.config = CONFIGS[method] self.basic_configs = basic_configs self.goal = basic_configs['goal'] self.method = method self.attack = ATTACKS[method](MODEL, 1) self._run = RUNS[method] self._optimized_configured = False def run(self, xs, ys, ys_target): xs_adv = np.zeros_like(xs) for lo in range(len(xs)): print(lo) s = slice(lo, lo + 1) xs_adv[s] = self._run(self, xs[s], ys[s], ys_target[s]) return xs_adv def _bin_search_nes(self, xs, ys, ys_target): found = np.array([False]) hi = np.zeros(shape=1, dtype=np.float32) + INIT_DISTORTION lo = np.zeros(shape=1, dtype=np.float32) xs_adv = np.zeros_like(xs) for i in range(10): print(i, hi, lo) if self.method == 'nes': specific_config = { 'magnitude': hi, 'learning_rate': hi * 0.15, 'min_lr': hi * 0.015 } elif self.method == 'spsa': specific_config = { 'magnitude': hi, 'learning_rate': hi * SPSA_LR_MUL } elif self.method == 'nattack': specific_config = {'magnitude': hi} self.attack.config(**specific_config, **self.basic_configs, **self.config) xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION) ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_}) flag = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys cond = np.logical_and(np.logical_not(found), flag) xs_adv[cond] = xs_adv_[cond] found[cond] = True lo[np.logical_not(found)] = hi[np.logical_not(found)] hi[np.logical_not(found)] *= 2.0 if found.all(): break for i in range(BIN_SEARCH_STEPS): mi = (lo + hi) / 2 print(i, mi) if self.method == 'nes': specific_config = { 'magnitude': mi, 'learning_rate': mi * 0.15, 'min_lr': mi * 0.015 } elif self.method == 'spsa': specific_config = { 'magnitude': mi, 'learning_rate': mi * SPSA_LR_MUL } elif self.method == 'nattack': specific_config = {'magnitude': mi} self.attack.config(**specific_config, **self.basic_configs, **self.config) xs_adv_ = self.attack.batch_attack(xs, ys, ys_target, SESSION) ys_adv_ = SESSION.run(LABELS, feed_dict={XS_PH: xs_adv_}) succ = ys_adv_ == ys_target if self.goal == 't' else ys_adv_ != ys not_succ = np.logical_not(succ) hi[succ] = mi[succ] lo[not_succ] = mi[not_succ] xs_adv[succ] = xs_adv_[succ] return xs_adv xs = np.load(args.xs).astype(MODEL.x_dtype.as_numpy_dtype) xs = (xs / 255.0) * (MODEL.x_max - MODEL.x_min) + MODEL.x_min ys = np.load(args.ys).astype(MODEL.y_dtype.as_numpy_dtype) ys_target = np.load(args.ys_target).astype(MODEL.y_dtype.as_numpy_dtype) output = args.output basic_configs = {'goal': goal, 'distance_metric': distance} benchmark = SuccessRateBenchmark(method, basic_configs) xs_adv = benchmark.run(xs, ys, ys_target) xs_adv = (xs_adv - MODEL.x_min) / (MODEL.x_max - MODEL.x_min) * 255.0 np.save(output, xs_adv)