def __init__(self, dataset, batch_size, targeted, target_type, epsilon, norm, lower_bound=0.0, upper_bound=1.0, max_queries=10000): """ :param epsilon: perturbation limit according to lp-ball :param norm: norm for the lp-ball constraint :param lower_bound: minimum value data point can take in any coordinate :param upper_bound: maximum value data point can take in any coordinate :param max_queries: max number of calls to model per data point :param max_crit_queries: max number of calls to early stopping criterion per data poinr """ assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.lower_bound = lower_bound self.upper_bound = upper_bound self._proj = None self.is_new_batch = False # self.early_stop_crit_fct = lambda model, x, y: 1 - model(x).max(1)[1].eq(y) self.targeted = targeted self.target_type = target_type self.dataset_loader = DataLoaderMaker.get_test_attacked_data(dataset, batch_size) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like(self.query_all) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all) self.lowest_change_ratio = 0.05 # the percentage of pixels that changes
def __init__(self, dataset, batch_size, targeted, target_type, epsilon, norm, lower_bound=0.0, upper_bound=1.0, max_queries=10000): assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.lower_bound = lower_bound self.upper_bound = upper_bound self.targeted = targeted self.target_type = target_type self.dataset_loader = DataLoaderMaker.get_test_attacked_data( dataset, batch_size) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all)
def __init__(self, dataset, batch_size, targeted, target_type, epsilon, norm, lower_bound=0.0, upper_bound=1.0, max_queries=10000, surrogate_model_names=None): assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.lower_bound = lower_bound self.upper_bound = upper_bound self.targeted = targeted self.target_type = target_type self.dataset_loader = DataLoaderMaker.get_test_attacked_data(dataset, batch_size) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like(self.query_all) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all) # self.cos_similarity_all = torch.zeros(self.total_images, max_queries) # N, T self.increase_loss_from_last_iter_with_1st_model_grad_record_all = OrderedDict() self.increase_loss_from_last_iter_with_2nd_model_grad_record_all = OrderedDict() self.increase_loss_from_last_iter_after_switch_record_all = OrderedDict() self.surrogate_model_record_all = OrderedDict() self.loss_x_pos_temp_record_all = OrderedDict() self.loss_x_neg_temp_record_all = OrderedDict() self.loss_after_switch_grad_record_all = OrderedDict() self.surrogate_model_names = surrogate_model_names
def __init__(self, args, directions_generator): self.rank_transform = not args.no_rank_transform self.random_mask = args.random_mask self.image_split = args.image_split self.sub_num_sample = args.sub_num_sample self.sigma = args.sigma self.starting_eps = args.starting_eps self.eps = args.epsilon self.sample_per_draw = args.sample_per_draw self.directions_generator = directions_generator self.max_iter = args.max_queries self.delta_eps = args.delta_eps self.max_lr = args.max_lr self.min_lr = args.min_lr self.targeted = args.targeted self.norm = args.norm self.dataset_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, 1) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all) self.dataset_name = args.dataset
def __init__(self, args, dataset, targeted, target_type, epsilon, norm, lower_bound=0.0, upper_bound=1.0, max_queries=10000): """ :param epsilon: perturbation limit according to lp-ball :param norm: norm for the lp-ball constraint :param lower_bound: minimum value data point can take in any coordinate :param upper_bound: maximum value data point can take in any coordinate :param max_queries: max number of calls to model per data point :param max_crit_queries: max number of calls to early stopping criterion per data poinr """ assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.lower_bound = lower_bound self.upper_bound = upper_bound # self.early_stop_crit_fct = lambda model, x, y: 1 - model(x).max(1)[1].eq(y) self.targeted = targeted self.target_type = target_type self.data_loader = DataLoaderMaker.get_test_attacked_data(dataset, args.batch_size) self.total_images = len(self.data_loader.dataset) self.att_iter = args.max_queries self.correct_all = torch.zeros(self.total_images) # number of images self.not_done_all = torch.zeros(self.total_images) # always set to 0 if the original image is misclassified self.success_all = torch.zeros(self.total_images) self.not_done_prob_all = torch.zeros(self.total_images) self.stop_iter_all = torch.zeros(self.total_images) self.ord = args.norm # linf, l1, l2 self.clip_min = args.clip_min self.clip_max = args.clip_max self.lr = args.lr self.beta1 = args.beta1 self.loss_fn = nn.CrossEntropyLoss().cuda()
def __init__(self, dataset_name, model, surrogate_model, meta_model, targeted, target_type, meta_predict_steps, finetune_times, finetune_lr): self.dataset_name = dataset_name self.data_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, 1) self.image_height = IMAGE_SIZE[self.dataset_name][0] self.image_width = IMAGE_SIZE[self.dataset_name][1] self.in_channels = IN_CHANNELS[self.dataset_name] self.model = model self.surrogate_model = surrogate_model self.model.cuda().eval() self.surrogate_model.cuda().eval() self.targeted = targeted # only support untargeted attack now self.target_type = target_type self.clip_min = 0.0 self.clip_max = 1.0 self.meta_predict_steps = meta_predict_steps self.finetune_times = finetune_times self.meta_model_for_q1 = meta_model self.meta_model_for_q2 = copy.deepcopy(meta_model) self.finetune_lr = finetune_lr self.pretrained_meta_weights = self.meta_model_for_q1.state_dict( ).copy() self.meta_optimizer_q1 = Adam(self.meta_model_for_q1.parameters(), lr=self.finetune_lr) self.meta_optimizer_q2 = Adam(self.meta_model_for_q2.parameters(), lr=self.finetune_lr) self.mse_loss = nn.MSELoss(reduction="mean")
def __init__(self, dataset, targeted, target_type, epsilon, norm, batch_size, lower_bound=0.0, upper_bound=1.0, max_queries=10000, max_crit_queries=np.inf): """ :param epsilon: perturbation limit according to lp-ball :param norm: norm for the lp-ball constraint :param lower_bound: minimum value data point can take in any coordinate :param upper_bound: maximum value data point can take in any coordinate :param max_queries: max number of calls to model per data point :param max_crit_queries: max number of calls to early stopping criterion per data poinr """ assert norm in ['linf', 'l2'], "{} is not supported".format(norm) assert not (np.isinf(max_queries) and np.isinf(max_crit_queries) ), "one of the budgets has to be finite!" self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.max_crit_queries = max_crit_queries self.best_est_deriv = None self.xo_t = None self.sgn_t = None self.h = np.zeros(batch_size).astype(np.int32) self.i = np.zeros(batch_size).astype(np.int32) self.exhausted = [False for _ in range(batch_size)] self.lower_bound = lower_bound self.upper_bound = upper_bound self._proj = None self.is_new_batch = False # self.early_stop_crit_fct = lambda model, x, y: 1 - model(x).max(1)[1].eq(y) self.targeted = targeted self.target_type = target_type self.data_loader = DataLoaderMaker.get_test_attacked_data( dataset, args.batch_size) self.total_images = len(self.data_loader.dataset) self.image_height = IMAGE_SIZE[dataset][0] self.image_width = IMAGE_SIZE[dataset][1] self.in_channels = IN_CHANNELS[dataset] self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all)
def __init__(self, args): self.dataset_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, args.batch_size) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like(self.query_all) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_loss_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all)
def __init__(self, dataset, batch_size, pixel_attack, freq_dims, stride, order, max_iters, targeted, target_type, norm, pixel_epsilon, l2_bound, linf_bound, lower_bound=0.0, upper_bound=1.0): """ :param pixel_epsilon: perturbation limit according to lp-ball :param norm: norm for the lp-ball constraint :param lower_bound: minimum value data point can take in any coordinate :param upper_bound: maximum value data point can take in any coordinate :param max_crit_queries: max number of calls to early stopping criterion per data poinr """ assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.pixel_epsilon = pixel_epsilon self.dataset = dataset self.norm = norm self.pixel_attack = pixel_attack self.lower_bound = lower_bound self.upper_bound = upper_bound self.freq_dims = freq_dims self.stride = stride self.order = order self.linf_bound = linf_bound self.l2_bound = l2_bound # self.early_stop_crit_fct = lambda model, x, y: 1 - model(x).max(1)[1].eq(y) self.max_iters = max_iters self.targeted = targeted self.target_type = target_type self.data_loader = DataLoaderMaker.get_test_attacked_data( dataset, batch_size) self.total_images = len(self.data_loader.dataset) self.image_height = IMAGE_SIZE[dataset][0] self.image_width = IMAGE_SIZE[dataset][1] self.in_channels = IN_CHANNELS[dataset] self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all)
def __init__(self, dataset_name, targeted): self.dataset_name = dataset_name self.num_classes = CLASS_NUM[self.dataset_name] self.dataset_loader = DataLoaderMaker.get_test_attacked_data( dataset_name, 1) self.total_images = len(self.dataset_loader.dataset) self.targeted = targeted self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all)
def __init__(self, dataset_name, model, surrogate_model, targeted, target_type): self.dataset_name = dataset_name self.data_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, 1) self.image_height = IMAGE_SIZE[self.dataset_name][0] self.image_width =IMAGE_SIZE[self.dataset_name][1] self.in_channels = IN_CHANNELS[self.dataset_name] self.model = model self.surrogate_model = surrogate_model self.model.cuda().eval() self.surrogate_model.cuda().eval() self.targeted = targeted # only support untargeted attack now self.target_type = target_type self.clip_min = 0.0 self.clip_max = 1.0
def __init__(self, function, config, device): self.config = config self.batch_size = config['batch_size'] self.function = function self.model = function.model self.device = device self.epsilon = self.config['epsilon'] self.gp = attack_bayesian_EI.Attack( f=self, dim=4, max_evals=1000, verbose=True, use_ard=True, max_cholesky_size=2000, n_training_steps=30, device=device, dtype="float32", ) self.query_limit = self.config['query_limit'] self.max_iters = self.config['max_iters'] self.init_iter = self.config["init_iter"] self.init_batch = self.config["init_batch"] self.memory_size = self.config["memory_size"] self.channels = self.config["channels"] self.image_height = self.config["image_height"] self.image_width = self.config["image_width"] self.gp_emptyX = torch.zeros((1, 4), device=device) self.gp_emptyfX = torch.zeros((1), device=device) self.local_forget_threshold = self.config['local_forget_threshold'] self.lr = self.config['lr'] self.dataset_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, args.batch_size) self.total_images = len(self.dataset_loader.dataset) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.maximum_queries = self.config["max_queries"]
def __init__(self, pop_size=5, generations=1000, cross_rate=0.7, mutation_rate=0.001, max_queries=2000, epsilon=8. / 255, iters=10, ensemble_models=None, targeted=False): self.loss_fn = nn.CrossEntropyLoss() self.dataset_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, 1) self.total_images = len(self.dataset_loader.dataset) # parameters about evolution algorithm self.pop_size = pop_size self.generations = generations self.cross_rate = cross_rate self.mutation_rate = mutation_rate # parameters about attack self.epsilon = epsilon self.clip_min = 0 self.clip_max = 1 # ensemble MI-FGSM parameters, use ensemble MI-FGSM attack generate adv as initial population self.ensemble_models = ensemble_models self.iters = iters self.targeted = targeted self.max_queries = max_queries self.idx = np.random.choice(np.arange(self.pop_size), size=2, replace=False) self.is_change = np.zeros(self.pop_size) self.pop_fitness = np.zeros(self.pop_size) self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all)
def __init__(self, dataset_name, targeted): self.dataset_name = dataset_name self.num_classes = CLASS_NUM[self.dataset_name] self.dataset_loader = DataLoaderMaker.get_test_attacked_data( dataset_name, 1) log.info("label index dict data build begin") # if self.dataset_name == "TinyImageNet": # self.candidate_loader = DataLoaderMaker.get_candidate_attacked_data(dataset_name, 1) # self.dataset = self.candidate_loader.dataset # else: self.dataset = self.dataset_loader.dataset self.label_data_index_dict = self.get_label_dataset(self.dataset) log.info("label index dict data build over!") self.total_images = len(self.dataset_loader.dataset) self.targeted = targeted self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all)
test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) data_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, 1) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model)
arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() dataset_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, args.batch_size) success_all = torch.zeros(dataset_loader.dataset.__len__()).float() correct_all = torch.zeros(dataset_loader.dataset.__len__()).float() query_all = torch.zeros(dataset_loader.dataset.__len__()).float() for batch_idx, data_tuple in enumerate(dataset_loader): if args.dataset == "ImageNet": if model.input_size[-1] >= 299: images, true_labels = data_tuple[1], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[1] if args.targeted: target_labels = torch.fmod(true_labels + 1, CLASS_NUM[args.dataset]) else:
def main(): args = get_args() with open(args.config) as config_file: state = json.load(config_file)["attack"][args.targeted] state = SimpleNamespace(**state) if args.save_prefix is not None: state.save_prefix = args.save_prefix if args.arch is not None: state.arch = args.arch if args.test_archs is not None: state.test_archs = args.test_archs state.OSP = args.OSP state.targeted = args.targeted device = torch.device(args.device if torch.cuda.is_available() else "cpu") targeted_str = "untargeted" if not state.targeted else "targeted" if state.targeted: save_name = "{}/train_pytorch_model/TREMBA/{}_{}_generator.pth.tar".format( PY_ROOT, args.dataset, targeted_str) else: save_name = "{}/train_pytorch_model/TREMBA/{}_{}_generator.pth.tar".format( PY_ROOT, args.dataset, targeted_str) weight = torch.load(save_name, map_location=device)["state_dict"] data_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, args.batch_size) encoder_weight = {} decoder_weight = {} for key, val in weight.items(): if key.startswith('0.'): encoder_weight[key[2:]] = val elif key.startswith('1.'): decoder_weight[key[2:]] = val archs = [] if args.test_archs: if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) args.arch = ",".join(archs) else: archs.append(args.arch) args.exp_dir = get_exp_dir_name(args.dataset, args.norm, args.targeted, args.target_type, args) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue if args.OSP: if state.source_model_name == "Adv_Denoise_Resnet152": source_model = resnet152_denoise() loaded_state_dict = torch.load((os.path.join( '{}/train_pytorch_model/TREMBA'.format(PY_ROOT), state.source_model_name + ".pth.tar"))) source_model.load_state_dict(loaded_state_dict) mean = np.array([0.485, 0.456, 0.406]) std = np.array([0.229, 0.224, 0.225]) # FIXME 仍然要改改 source_model = nn.Sequential(Normalize(mean, std), source_model) source_model.to(device) source_model.eval() if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.eval() encoder = ImagenetEncoder() decoder = ImagenetDecoder(args.dataset) encoder.load_state_dict(encoder_weight) decoder.load_state_dict(decoder_weight) model.to(device) encoder.to(device) encoder.eval() decoder.to(device) decoder.eval() F = Function(model, state.batch_size, state.margin, CLASS_NUM[args.dataset], state.targeted) total_success = 0 count_total = 0 queries = [] not_done = [] correct_all = [] for i, (images, labels) in enumerate(data_loader): images = images.to(device) labels = labels.to(device) logits = model(images) correct = torch.argmax(logits, dim=1).eq(labels).item() correct_all.append(int(correct)) if correct: if args.targeted: if args.target_type == 'random': target_labels = torch.randint( low=0, high=CLASS_NUM[args.dataset], size=labels.size()).long().cuda() invalid_target_index = target_labels.eq(labels) while invalid_target_index.sum().item() > 0: target_labels[ invalid_target_index] = torch.randint( low=0, high=logit.shape[1], size=target_labels[invalid_target_index]. shape).long().cuda() invalid_target_index = target_labels.eq(labels) elif args.target_type == 'least_likely': with torch.no_grad(): logit = model(images) target_labels = logit.argmin(dim=1) elif args.target_type == "increment": target_labels = torch.fmod( labels + 1, CLASS_NUM[args.dataset]).cuda() labels = target_labels[0].item() else: labels = labels[0].item() if args.OSP: hinge_loss = MarginLossSingle(state.white_box_margin, state.target) images.requires_grad = True latents = encoder(images) for k in range(state.white_box_iters): perturbations = decoder(latents) * state.epsilon logits = source_model( torch.clamp(images + perturbations, 0, 1)) loss = hinge_loss(logits, labels) grad = torch.autograd.grad(loss, latents)[0] latents = latents - state.white_box_lr * grad with torch.no_grad(): success, adv, query_count = EmbedBA( F, encoder, decoder, images[0], labels, state, latents.view(-1)) else: with torch.no_grad(): success, adv, query_count = EmbedBA( F, encoder, decoder, images[0], labels, state) not_done.append(1 - int(success)) total_success += int(success) count_total += int(correct) if success: queries.append(query_count) else: queries.append(args.max_queries) log.info( "image: {} eval_count: {} success: {} average_count: {} success_rate: {}" .format(i, F.current_counts, success, F.get_average(), float(total_success) / float(count_total))) F.new_counter() else: queries.append(0) not_done.append(1) log.info("The {}-th image is already classified incorrectly.". format(i)) correct_all = np.concatenate(correct_all, axis=0).astype(np.int32) query_all = np.array(queries).astype(np.int32) not_done_all = np.array(not_done).astype(np.int32) success = (1 - not_done_all) * correct_all success_query = success * query_all meta_info_dict = { "query_all": query_all.tolist(), "not_done_all": not_done_all.tolist(), "correct_all": correct_all.tolist(), "mean_query": np.mean(success_query[np.nonzero(success)[0]]).item(), "max_query": np.max(success_query[np.nonzero(success)[0]]).item(), "median_query": np.median(success_query[np.nonzero(success)[0]]).item(), "avg_not_done": np.mean(not_done_all[np.nonzero(correct_all)[0]].astype( np.float32)).item(), "args": vars(args) } with open(save_result_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) log.info("Done, write stats info to {}".format(save_result_path))
def main(args, result_dir_path): log.info('Loading %s model and test data' % args.dataset) dataset_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, 1) if args.test_archs: archs = [] if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] if args.attack_defense: meta_model_path = '{}/train_pytorch_model/meta_grad_regression/{}_without_resnet.pth.tar'.format( PY_ROOT, args.dataset) else: meta_model_path = '{}/train_pytorch_model/meta_grad_regression/{}.pth.tar'.format( PY_ROOT, args.dataset) assert os.path.exists(meta_model_path), "{} does not exist!".format( meta_model_path) meta_model = load_meta_model(meta_model_path) log.info("Load meta model from {}".format(meta_model_path)) attack = MetaGradAttack(args, norm=args.norm, epsilon=args.epsilon, targeted=args.targeted, search_steps=args.binary_steps, max_steps=args.maxiter, use_log=not args.use_zvalue, cuda=not args.no_cuda) for arch in archs: if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda().eval() query_all = [] not_done_all = [] correct_all = [] img_no = 0 total_success = 0 l2_total = 0.0 avg_step = 0 avg_time = 0 avg_qry = 0 if args.attack_defense: result_dump_path = result_dir_path + "/{}_{}_result.json".format( arch, args.defense_model) else: result_dump_path = result_dir_path + "/{}_result.json".format(arch) # if os.path.exists(result_dump_path): # continue log.info("Begin attack {} on {}".format(arch, args.dataset)) for i, data_tuple in enumerate(dataset_loader): if args.dataset == "ImageNet": if model.input_size[-1] >= 299: img, true_labels = data_tuple[1], data_tuple[2] else: img, true_labels = data_tuple[0], data_tuple[2] else: img, true_labels = data_tuple[0], data_tuple[1] args.init_size = model.input_size[-1] if img.size(-1) != model.input_size[-1]: img = F.interpolate(img, size=model.input_size[-1], mode='bilinear', align_corners=True) img, true_labels = img.to(0), true_labels.to(0) with torch.no_grad(): pred_logit = model(img) pred_label = pred_logit.argmax(dim=1) correct = pred_label.eq(true_labels).detach().cpu().numpy().astype( np.int32) correct_all.append(correct) if pred_label[0].item() != true_labels[0].item(): log.info( "Skip wrongly classified image no. %d, original class %d, classified as %d" % (i, pred_label.item(), true_labels.item())) query_all.append(0) not_done_all.append( 1) # 原本就分类错误,not_done给1,假如99%都原本分类错误的话, avg_not_done = 99% continue img_no += 1 timestart = time.time() meta_model_copy = copy.deepcopy(meta_model) if args.targeted: if args.target_type == 'random': target_labels = torch.randint( low=0, high=CLASS_NUM[args.dataset], size=true_labels.size()).long().cuda() invalid_target_index = target_labels.eq(true_labels) while invalid_target_index.sum().item() > 0: target_labels[invalid_target_index] = torch.randint( low=0, high=pred_logit.shape[1], size=target_labels[invalid_target_index].shape ).long().cuda() invalid_target_index = target_labels.eq(true_labels) elif args.target_type == 'least_likely': target_labels = pred_logit.argmin(dim=1) elif args.target_type == "increment": target_labels = torch.fmod(true_labels + 1, CLASS_NUM[args.dataset]) else: raise NotImplementedError('Unknown target_type: {}'.format( args.target_type)) else: target_labels = None target = true_labels if not args.targeted else target_labels adv, const, first_step, success_queries = attack.run( model, meta_model_copy, img, target) timeend = time.time() if len(adv.shape) == 3: adv = adv.reshape((1, ) + adv.shape) adv = torch.from_numpy(adv).permute(0, 3, 1, 2).cuda() # BHWC -> BCHW diff = (adv - img).detach().cpu().numpy() l2_distortion = np.sqrt(np.sum(np.square(diff))).item() with torch.no_grad(): adv_pred_logit = model(adv) adv_pred_label = adv_pred_logit.argmax(dim=1) success = False if not args.targeted: # target is true label if adv_pred_label[0].item() != target[0].item(): success = True else: if adv_pred_label[0].item() == target[0].item(): success = True if success_queries > args.max_queries: success = False if success: # (first_step-1)//args.finetune_intervalargs.update_pixels2+first_step # The first step is the iterations used that find the adversarial examples; # args.finetune_interval is the finetuning per iterations; # args.update_pixels is the queried pixels each iteration. # Currently, we find only f(x+h)-f(x) could estimate the gradient well, so args.update_pixels*1 in my updated codes. not_done_all.append(0) # only 1 query for i pixle, because the estimated function is f(x+h)-f(x)/h query_all.append(success_queries) total_success += 1 l2_total += l2_distortion avg_step += first_step avg_time += timeend - timestart # avg_qry += (first_step-1)//args.finetune_interval*args.update_pixels*1+first_step avg_qry += success_queries log.info("Attack {}-th image: {}, query:{}".format( i, "success", success_queries)) else: not_done_all.append(1) query_all.append(args.max_queries) log.info("Attack {}-th image: {}, query:{}".format( i, "fail", success_queries)) model.cpu() if total_success != 0: log.info( "[STATS][L1] total = {}, time = {:.3f}, distortion = {:.5f}, avg_step = {:.5f},avg_query = {:.5f}, success_rate = {:.3f}" .format(img_no, avg_time / total_success, l2_total / total_success, avg_step / total_success, avg_qry / total_success, total_success / float(img_no))) correct_all = np.concatenate(correct_all, axis=0).astype(np.int32) query_all = np.array(query_all).astype(np.int32) not_done_all = np.array(not_done_all).astype(np.int32) success = (1 - not_done_all) * correct_all success_query = success * query_all # query_all_bounded = query_all.copy() # not_done_all_bounded = not_done_all.copy() # out_of_bound_indexes = np.where(query_all_bounded > args.max_queries)[0] # if len(out_of_bound_indexes) > 0: # not_done_all_bounded[out_of_bound_indexes] = 1 # success_bounded = (1-not_done_all_bounded) * correct_all # success_query_bounded = success_bounded * query_all_bounded # # query_threshold_success_rate_bounded, query_success_rate_bounded = success_rate_and_query_coorelation(query_all_bounded, not_done_all_bounded) # success_rate_to_avg_query_bounded = success_rate_avg_query(query_all_bounded, not_done_all_bounded) meta_info_dict = { "query_all": query_all.tolist(), "not_done_all": not_done_all.tolist(), "correct_all": correct_all.tolist(), "mean_query": np.mean(success_query[np.nonzero(success)[0]]).item(), "max_query": np.max(success_query[np.nonzero(success)[0]]).item(), "median_query": np.median(success_query[np.nonzero(success)[0]]).item(), "avg_not_done": np.mean( not_done_all.astype( np.float32)[np.nonzero(correct_all)[0]]).item(), # "mean_query_bounded_max_queries": np.mean(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "max_query_bounded_max_queries": np.max(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "median_query_bounded_max_queries": np.median(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "avg_not_done_bounded_max_queries": np.mean(not_done_all_bounded.astype(np.float32)).item(), # "query_threshold_success_rate_dict_bounded": query_threshold_success_rate_bounded, # "query_success_rate_dict_bounded": query_success_rate_bounded, # "success_rate_to_avg_query_bounded": success_rate_to_avg_query_bounded, "args": vars(args) } with open(result_dump_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) log.info("done, write stats info to {}".format(result_dump_path))
def main(): parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') parser.add_argument('--arch', type=str, required=True, help="The arch used to generate adversarial images for testing") parser.add_argument("--gpu",type=str,required=True) parser.add_argument("--dataset", type=str, required=True) parser.add_argument('--test_mode', default=0, type=int, choices=list(range(10))) # parser.add_argument('--model', default='res', type=str) parser.add_argument('--n_epoch', default=200, type=int) parser.add_argument('--batch_size', default=128, type=int) parser.add_argument('--test_batch_size', default=10, type=int) parser.add_argument('--lambd', default=0.0001, type=float) parser.add_argument('--noise_dev', default=20.0, type=float) parser.add_argument('--Linfinity', default=8/255, type=float) parser.add_argument('--binary_threshold', default=0.5, type=float) parser.add_argument('--lr_mode', default=0, type=int) parser.add_argument('--test_interval', default=100, type=int) parser.add_argument('--lr', default=1e-3, type=float) parser.add_argument("--use_res_net",action="store_true") args = parser.parse_args() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) cudnn.deterministic = True model_path = '{}/train_pytorch_model/adversarial_train/com_defend/{}@{}@epoch_{}@batch_{}.pth.tar'.format( PY_ROOT, args.dataset, args.arch, args.n_epoch, args.batch_size) os.makedirs(os.path.dirname(model_path), exist_ok=True) set_log_file(os.path.dirname(model_path) + "/train_{}.log".format(args.dataset)) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info('Called with args:') print_args(args) in_channels = IN_CHANNELS[args.dataset] # if args.use_res_net: # if args.test_mode == 0: # com_defender = ModelRes(in_channels=in_channels, com_disable=True,rec_disable=True) # args.save_model = 'normal' # elif args.test_mode == 1: # com_defender = ModelRes(in_channels=in_channels, n_com=1,n_rec=3,com_disable=False,rec_disable=True) # args.save_model = '1_on_off' # elif args.test_mode == 2: # com_defender = ModelRes(in_channels=in_channels, n_com=2,n_rec=3,com_disable=False,rec_disable=True) # args.save_model = '2_on_off' # elif args.test_mode == 3: # com_defender = ModelRes(in_channels=in_channels, n_com=3,n_rec=3,com_disable=False,rec_disable=True) # args.save_model = '3_on_off' # elif args.test_mode == 4: # com_defender = ModelRes(in_channels=in_channels, n_com=3,n_rec=1,com_disable=True,rec_disable=False) # args.save_model = 'off_on_1' # elif args.test_mode == 5: # com_defender = ModelRes(in_channels=in_channels, n_com=3,n_rec=2,com_disable=True,rec_disable=False) # args.save_model = 'off_on_2' # elif args.test_mode == 6: # com_defender = ModelRes(in_channels=in_channels, n_com=3,n_rec=3,com_disable=True,rec_disable=False) # args.save_model = 'off_on_3' # elif args.test_mode == 7: # com_defender = ModelRes(in_channels=in_channels, n_com=1,n_rec=1,com_disable=False,rec_disable=False) # args.save_model = '1_1' # elif args.test_mode == 8: # com_defender = ModelRes(in_channels=in_channels, n_com=2,n_rec=2,com_disable=False,rec_disable=False) # args.save_model = '2_2' # elif args.test_mode == 9: # com_defender = ModelRes(in_channels=in_channels, n_com=3,n_rec=3,com_disable=False,rec_disable=False) # args.save_model = '3_3' # else: com_defender = ComDefend(in_channels, args.noise_dev) args.save_model = "normal_network" log.info('test mode: {}, model name: {}'.format(args.test_mode, args.save_model)) if args.gpu is not None: log.info("Use GPU: {} for training".format(args.gpu)) log.info("=> creating model '{}'".format(args.arch)) log.info("after train, model will be saved to {}".format(model_path)) com_defender.cuda() cudnn.benchmark = True train_loader = DataLoaderMaker.get_imgid_img_label_data_loader(args.dataset, args.batch_size, True, seed=1234) test_attack_dataset_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, args.batch_size) log.info("Begin generate the adversarial examples.") target_model, adv_images, adv_true_labels = test_attack(args.Linfinity, args.arch, args.dataset, test_attack_dataset_loader) # 这些图片被用来验证 log.info("Generate adversarial examples done!") best_acc = torch.zeros(1) for epoch in range(0, args.n_epoch): train(args, train_loader, com_defender, epoch, target_model, adv_images, adv_true_labels,best_acc, model_path)
def main(target_model, result_dump_path, args): # make model(s) log.info('Initializing target model {} on {}'.format(args.arch, args.dataset)) ref_models = OrderedDict() for i, ref_arch in enumerate(args.ref_arch): params = dict() params['train_data'] = args.ref_arch_train_data params['epoch'] = args.ref_arch_epoch log.info('Initializing ref model {} on {} ({} of {}), params: {}'.format( ref_arch, args.dataset, i + 1, len(args.ref_arch), params)) ref_models[ref_arch] = StandardModel(args.dataset, ref_arch, no_grad=False, is_subspace_attack_ref_arch=True, ref_arch_train_data=args.ref_arch_train_data, ref_arch_epoch=args.ref_arch_epoch).cuda().eval() log.info('All target_models have been initialized, including 1 target model and {} ref target_models'.format(len(args.ref_arch))) # make loader loader = DataLoaderMaker.get_test_attacked_data(args.dataset, args.batch_size) total_images = len(loader.dataset) # make operators prior_step = eval('{}_prior_step'.format(args.prior_update)) image_step = eval('{}_image_step'.format(args.norm_type)) proj_step = eval('{}_proj_step'.format(args.norm_type)) if args.delta_size > 0: # resize upsampler = lambda x: F.interpolate(x, size=target_model.input_size[-1], mode='bilinear', align_corners=True) # 这就是低维度的sub space放大回来 downsampler = lambda x: F.interpolate(x, size=args.delta_size, mode='bilinear', align_corners=True) else: # no resize, upsampler = downsampler = identity upsampler = downsampler = lambda x: x # CIFAR-10不用缩尺寸, ImageNet需要缩尺寸 # make loss function loss_func = eval('{}_loss'.format(args.loss)) # init arrays for saving results query_all = torch.zeros(total_images) correct_all = torch.zeros_like(query_all) not_done_all = torch.zeros_like(query_all) # always set to 0 if the original image is misclassified success_all = torch.zeros_like(query_all) success_query_all = torch.zeros_like(query_all) not_done_loss_all = torch.zeros_like(query_all) not_done_prob_all = torch.zeros_like(query_all) # make directory for saving results result_dirname = osp.join(args.exp_dir, 'results') os.makedirs(result_dirname, exist_ok=True) # fixed direction for illustration experiments if args.num_fix_direction > 0: if len(args.ref_arch) == 0: # fixed random direction assert args.dataset == 'CIFAR-10' state = np.random.get_state() np.random.seed(args.fix_direction_seed) fix_direction = np.random.randn(3072, *target_model.input_size)[:args.num_fix_direction] np.random.set_state(state) fix_direction = np.ascontiguousarray(fix_direction) fix_direction = torch.FloatTensor(fix_direction).to(device) else: # fixed gradient direction (calculated at clean inputs) assert args.num_fix_direction == len(args.ref_arch) for batch_index, data_tuple in enumerate(loader): if args.dataset == "ImageNet": if target_model.input_size[-1] >= 299: image, label = data_tuple[1], data_tuple[2] else: image, label = data_tuple[0], data_tuple[2] else: image, label = data_tuple[0], data_tuple[1] if image.size(-1) != target_model.input_size[-1]: image = F.interpolate(image, size=target_model.input_size[-1], mode='bilinear') assert image.max().item() <= 1 assert image.min().item() >= 0 # move image and label to device image = image.to(device) label = label.to(device) adv_image = image.clone() # get logit and prob logit = target_model(image) adv_logit = logit.clone() pred = logit.argmax(dim=1) # choose target classes for targeted attack if args.targeted: if args.target_type == 'random': target = torch.randint(low=0, high=logit.shape[1], size=label.shape).long().to(device) # make sure target is not equal to label for any example invalid_target_index = target.eq(label) while invalid_target_index.sum().item() > 0: target[invalid_target_index] = torch.randint(low=0, high=logit.shape[1], size=target[invalid_target_index].shape).long().to( device) invalid_target_index = target.eq(label) elif args.target_type == 'least_likely': target = logit.argmin(dim=1) elif args.target_type == "increment": target = torch.fmod(label+1, CLASS_NUM[args.dataset]) else: raise NotImplementedError('Unknown target_type: {}'.format(args.target_type)) else: target = None # init masks and selectors correct = pred.eq(label).float() query = torch.zeros(args.batch_size).to(device) not_done = correct.clone() selected = torch.arange(batch_index * args.batch_size, (batch_index + 1) * args.batch_size) # init prior if args.delta_size > 0: prior = torch.zeros(args.batch_size, target_model.input_size[0], args.delta_size, args.delta_size).to(device) else: prior = torch.zeros(args.batch_size, *target_model.input_size).to(device) # perform attack for step_index in range(args.max_queries // 2): # increase query counts query = query + 2 * not_done # not_done = 预测与gt label相等的图片个数 # calculate drop probability if step_index < args.ref_arch_drop_grow_iter: drop = args.ref_arch_init_drop else: # args.ref_arch_max_drop 默认等于0.5 drop = args.ref_arch_max_drop - \ (args.ref_arch_max_drop - args.ref_arch_init_drop) * \ np.exp(-(step_index - args.ref_arch_drop_grow_iter) * args.ref_arch_drop_gamma) # finite difference for gradient estimation if len(ref_models) > 0: # select ref model to calculate gradient # selected_ref_arch_index = torch.randint(low=0, high=len(ref_models), size=(1,)).long().item() # get original model logit's grad adv_logit = adv_logit.detach() adv_logit.requires_grad = True loss = loss_func(adv_logit, label, target).mean() logit_grad = torch.autograd.grad(loss, [adv_logit])[0] # calculate gradient for all ref target_models def calc_ref_grad(adv_image_, ref_model_, drop_=0): adv_image_ = adv_image_.detach() adv_image_.requires_grad = True if adv_image_.grad: adv_image_.grad[:] = 0. ref_model_.zero_grad() # assign dropout probability ref_model_.drop = drop_ # 这个可以进模型代码看看,drop怎么做的 # forward ref model if ref_model_.input_size != model.input_size: ref_logit_ = ref_model_(F.interpolate(adv_image_, size=ref_model_.input_size[-1], mode='bilinear', align_corners=True)) else: ref_logit_ = ref_model_(adv_image_) # backward ref model using logit_grad from the victim model ref_grad_ = torch.autograd.grad(ref_logit_, [adv_image_], grad_outputs=[logit_grad])[0] ref_grad_ = downsampler(ref_grad_) # 高维度缩小 # compute dl/dv if args.fix_grad: if prior.view(prior.shape[0], -1).norm(dim=1).min().item() > 0: # -1 / ||v|| ** 3 (||v|| ** 2 dL/dv - v(v^T dL/dv)) g1 = norm(prior) ** 2 * ref_grad_ g2 = prior * (prior * ref_grad_).sum(dim=(1, 2, 3)).view(-1, 1, 1, 1) ref_grad_ = g1 - g2 return ref_grad_ / norm(ref_grad_) # 拿到direction # calculate selected ref model's gradient if args.num_fix_direction == 0: # 随机选择一个模型,输入adv_image,得到梯度.这个梯度是否准确不知道,因为是随机选择的模型,不如用网络生成 direction = calc_ref_grad(adv_image, list(ref_models.values())[selected_ref_arch_index], drop_=drop) else: # for illustrate experiment in rebuttal assert args.loss == 'cw' assert drop == 0 direction = calc_ref_grad(image, list(ref_models.values())[selected_ref_arch_index], drop_=drop) else: # use random search direction solely if args.num_fix_direction > 0: # use fixed direction (for illustration experiments) if len(args.ref_arch) == 0: # fixed random direction # fix_direction.shape: [num_fix_direction, C, H, W] # coeff.shape: [num_Fix_direction, N] coeff = torch.randn(args.num_fix_direction, prior.shape[0]).to(device) direction = (fix_direction.view(fix_direction.shape[0], 1, *fix_direction.shape[1:]) * coeff.view(coeff.shape[0], coeff.shape[1], 1, 1, 1)).sum(dim=0) else: # fixed gradient direction (calculated at clean inputs) for rebuttal # direction has already been calculated assert direction.shape[0] == image.shape[0] else: direction = torch.randn_like(prior) # normalize search direction direction = direction / norm(direction) #这个方向是用随机选择一个model,估计出来的梯度,可以换成用meta预测方向。ground truth用prior来给最终loss,由于meta训练已知模型的话梯度非常好给,所以可以用梯度累积后的prior做ground truth q1 = upsampler(prior + args.exploration * direction) # 这两个upsampler和downsampler,使用在ImageNet大图起作用 q2 = upsampler(prior - args.exploration * direction) # 方向来自于随机选取的model,可以改为meta learning设计一下 l1 = loss_func(target_model(adv_image + args.fd_eta * q1 / norm(q1)), label, target) # 需要查询 l2 = loss_func(target_model(adv_image + args.fd_eta * q2 / norm(q2)), label, target) grad = (l1 - l2) / (args.fd_eta * args.exploration) # 需要2次查询,grad是论文Alg1第11行左边那个Delta_t,用于更新梯度的一个量 # 这段抄袭的bandit attack,但是把原来的exp_noise换成了direction grad = grad.view(-1, 1, 1, 1) * direction # grad.shape == direction.shape == prior.shape ?= image.shape 这就是精髓所在 # update prior,其实prior就是梯度,因为后一个prior和前一个有联系,类似贝叶斯的prior prior = prior_step(prior, grad, args.prior_lr) # 用grad更新prior,piror最后更新到图像上。prior就是图像梯度 # extract grad from prior grad = upsampler(prior) # prior相当于梯度 # update adv_image (correctly classified images only) adv_image = image_step(adv_image, grad * correct.view(-1, 1, 1, 1), args.image_lr) adv_image = proj_step(image, args.epsilon, adv_image) adv_image = torch.clamp(adv_image, 0, 1) # update statistics with torch.no_grad(): adv_logit = target_model(adv_image) adv_pred = adv_logit.argmax(dim=1) adv_prob = F.softmax(adv_logit, dim=1) adv_loss = loss_func(adv_logit, label, target) if args.targeted: not_done = not_done * (1 - adv_pred.eq(target)).float() else: not_done = not_done * adv_pred.eq(label).float() success = (1 - not_done) * correct # currently done, originally correct --> success success_query = success * query not_done_loss = adv_loss * not_done not_done_prob = adv_prob[torch.arange(args.batch_size), label] * not_done # log log.info('Attacking image {} - {} / {}, step {}, max query {}'.format( batch_index * args.batch_size, (batch_index + 1) * args.batch_size, total_images, step_index + 1, int(query.max().item()) )) log.info(' correct: {:.4f}'.format(correct.mean().item())) log.info(' not_done: {:.4f}'.format(not_done.mean().item())) log.info(' fd_scalar: {:.4f}'.format((l1 - l2).mean().item())) log.info(' drop: {:.4f}'.format(drop)) if success.sum().item() > 0: log.info(' mean_query: {:.4f}'.format(success_query[success.byte()].mean().item())) log.info(' median_query: {:.4f}'.format(success_query[success.byte()].median().item())) if not_done.sum().item() > 0: log.info(' not_done_loss: {:.4f}'.format(not_done_loss[not_done.byte()].mean().item())) log.info(' not_done_prob: {:.4f}'.format(not_done_prob[not_done.byte()].mean().item())) # early break if all succeed if not not_done.byte().any(): log.info(' image {} - {} all succeed in step {}, break'.format( batch_index * args.batch_size, (batch_index + 1) * args.batch_size, step_index )) break # save results to arrays # 下面这段代码统计最终的统计量,比如success_query_all for key in ['query', 'correct', 'not_done', 'success', 'success_query', 'not_done_loss', 'not_done_prob']: value_all = eval('{}_all'.format(key)) value = eval(key) value_all[selected] = value.detach().float().cpu() # log statistics for $total_images images log.info('Attack finished ({} images)'.format(total_images)) log.info(' avg correct: {:.4f}'.format(correct_all.mean().item())) log.info(' avg not_done: {:.4f}'.format(not_done_all.mean().item())) if success_all.sum().item() > 0: log.info(' avg mean_query: {:.4f}'.format(success_query_all[success_all.byte()].mean().item())) log.info(' avg median_query: {:.4f}'.format(success_query_all[success_all.byte()].median().item())) if not_done_all.sum().item() > 0: log.info(' avg not_done_loss: {:.4f}'.format(not_done_loss_all[not_done_all.byte()].mean().item())) log.info(' avg not_done_prob: {:.4f}'.format(not_done_prob_all[not_done_all.byte()].mean().item())) log.info('Saving results to {}'.format(result_dump_path)) meta_info_dict = {"avg_correct": correct_all.mean().item(), "avg_not_done": not_done_all[correct_all.byte()].mean().item(), "mean_query": success_query_all[success_all.byte()].mean().item(), "median_query": success_query_all[success_all.byte()].median().item(), "max_query": success_query_all[success_all.byte()].max().item(), "correct_all": correct_all.detach().cpu().numpy().astype(np.int32).tolist(), "not_done_all": not_done_all.detach().cpu().numpy().astype(np.int32).tolist(), "query_all": query_all.detach().cpu().numpy().astype(np.int32).tolist(), "not_done_loss": not_done_loss_all[not_done_all.byte()].mean().item(), "not_done_prob": not_done_prob_all[not_done_all.byte()].mean().item(), "args":vars(args)} with open(result_dump_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) # save results to disk log.info('Done')
def __init__(self, dataset, order, r, rho, mom, n_samples, targeted, target_type, norm, epsilon, low_dim, lower_bound=0.0, upper_bound=1.0, max_queries=10000): """ :param epsilon: perturbation limit according to lp-ball :param norm: norm for the lp-ball constraint :param lower_bound: minimum value data point can take in any coordinate :param upper_bound: maximum value data point can take in any coordinate :param max_queries: max number of calls to model per data point :param max_crit_queries: max number of calls to early stopping criterion per data poinr """ assert norm in ['linf', 'l2'], "{} is not supported".format(norm) self.epsilon = epsilon self.norm = norm self.max_queries = max_queries self.order = order self.r = r self.lower_bound = lower_bound self.upper_bound = upper_bound # self.early_stop_crit_fct = lambda model, x, y: 1 - model(x).max(1)[1].eq(y) self.targeted = targeted self.target_type = target_type self.dataset = dataset self.data_loader = DataLoaderMaker.get_test_attacked_data(dataset, 1) self.total_images = len(self.data_loader.dataset) self.image_height = IMAGE_SIZE[dataset][0] self.image_width = IMAGE_SIZE[dataset][1] self.in_channels = IN_CHANNELS[dataset] self.low_dim = low_dim self.query_all = torch.zeros(self.total_images) self.correct_all = torch.zeros_like(self.query_all) # number of images self.not_done_all = torch.zeros_like( self.query_all ) # always set to 0 if the original image is misclassified self.success_all = torch.zeros_like(self.query_all) self.success_query_all = torch.zeros_like(self.query_all) self.not_done_prob_all = torch.zeros_like(self.query_all) if dataset.startswith("CIFAR"): # TODO 实验不同的参数效果 self.freq_dim = 11 # 28 self.stride = 7 elif dataset == "TinyImageNet": self.freq_dim = 15 self.stride = 7 elif dataset == "ImageNet": self.freq_dim = 28 self.stride = 7 self.mom = mom # default 1 not add self.n_samples = n_samples # number of samples per iteration (1 by default), not the number of images to be evaluated. self.rho = rho self.construct_random_matrix()