def post_averged_model(dataset, arch): model = StandardModel(dataset, arch, no_grad=False) R = 30 if arch in ["resnet-101", "resnet-110", "resnet-152"] else 6 return PostAveragedNetwork(model, K=15, R=R, num_classes=CLASS_NUM[dataset])
def main(args, arch): model = StandardModel(args.dataset, arch, args.solver != "fake_zero") model.cuda() model.eval() if args.init_size is None: args.init_size = model.input_size[-1] log.info( "Argument init_size is not set and not using autoencoder, set to image original size:{}" .format(args.init_size)) target_str = "untargeted" if not args.targeted else "targeted_{}".format( args.target_type) save_result_path = args.exp_dir + "/data_{}@arch_{}@solver_{}@{}_result.json".format( args.dataset, arch, args.solver, target_str) if os.path.exists(save_result_path): model.cpu() return attack_framework = ZooAttackFramework(args) attack_framework.attack_dataset_images(args, arch, model, save_result_path) model.cpu()
def test_attack(threshold, arch, dataset, test_loader): target_model = StandardModel(dataset, arch, no_grad=False) if torch.cuda.is_available(): target_model = target_model.cuda() target_model.eval() attack = LinfPGDAttack(target_model, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=threshold, nb_iter=30, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0, targeted=False) all_count = 0 success_count = 0 all_adv_images = [] all_true_labels = [] for idx, (img, true_label) in enumerate(test_loader): img = img.cuda() true_label = true_label.cuda().long() adv_image = attack.perturb(img, true_label) # (3, 224, 224), float if adv_image is None: continue adv_label = target_model.forward(adv_image).max(1)[1].detach().cpu().numpy().astype(np.int32) # adv_image = np.transpose(adv_image, (0, 2, 3, 1)) # N,C,H,W -> (N, H, W, 3), float all_count += len(img) true_label_np = true_label.detach().cpu().numpy().astype(np.int32) success_count+= len(np.where(true_label_np != adv_label)[0]) all_adv_images.append(adv_image.cpu().detach().numpy()) all_true_labels.append(true_label_np) attack_success_rate = success_count / float(all_count) log.info("Before train. Attack success rate is {:.3f}".format(attack_success_rate)) return target_model, np.concatenate(all_adv_images,0), np.concatenate(all_true_labels, 0) # N,224,224,3
def main(args, arch): model = StandardModel(args["dataset"], arch, False) model.cuda() model.eval() # attack related settings if args["attack_method"] == "zoo" or args[ "attack_method"] == "autozoom_bilin": if args["img_resize"] is None: args["img_resize"] = model.input_size[-1] log.info( "Argument img_resize is not set and not using autoencoder, set to image original size:{}" .format(args["img_resize"])) codec = None if args["attack_method"] == "zoo_ae" or args[ "attack_method"] == "autozoom_ae": codec = Codec(model.input_size[-1], IN_CHANNELS[args["dataset"]], args["compress_mode"], args["resize"], use_tanh=args["use_tanh"]) codec.load_codec(args["codec_path"]) codec.cuda() decoder = codec.decoder args["img_resize"] = decoder.input_shape[1] log.info( "Loading autoencoder: {}, set the attack image size to:{}".format( args["codec_path"], args["img_resize"])) # setup attack if args["attack_method"] == "zoo": blackbox_attack = ZOO(model, args["dataset"], args) elif args["attack_method"] == "zoo_ae": blackbox_attack = ZOO_AE(model, args["dataset"], args, decoder) elif args["attack_method"] == "autozoom_bilin": blackbox_attack = AutoZOOM_BiLIN(model, args["dataset"], args) elif args["attack_method"] == "autozoom_ae": blackbox_attack = AutoZOOM_AE(model, args["dataset"], args, decoder) target_str = "untargeted" if args[ "attack_type"] != "targeted" else "targeted_{}".format( args["target_type"]) save_result_path = args[ "exp_dir"] + "/data_{}@arch_{}@attack_{}@{}_result.json".format( args["dataset"], arch, args["attack_method"], target_str) attack_framework = AutoZoomAttackFramework(args) attack_framework.attack_dataset_images(args, blackbox_attack, arch, model, codec, save_result_path) model.cpu()
def generate(datasetname, batch_size): save_dir_path = "{}/data_adv_defense/guided_denoiser".format(PY_ROOT) os.makedirs(save_dir_path, exist_ok=True) set_log_file(save_dir_path + "/generate_{}.log".format(datasetname)) data_loader = DataLoaderMaker.get_img_label_data_loader(datasetname, batch_size, is_train=True) attackers = [] for model_name in MODELS_TRAIN_STANDARD[datasetname] + MODELS_TEST_STANDARD[datasetname]: model = StandardModel(datasetname, model_name, no_grad=False) model = model.cuda().eval() linf_PGD_attack =LinfPGDAttack(model, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=0.031372, nb_iter=30, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0, targeted=False) l2_PGD_attack = L2PGDAttack(model, loss_fn=nn.CrossEntropyLoss(reduction="sum"),eps=4.6, nb_iter=30,clip_min=0.0, clip_max=1.0, targeted=False) FGSM_attack = FGSM(model, loss_fn=nn.CrossEntropyLoss(reduction="sum")) momentum_attack = MomentumIterativeAttack(model, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=0.031372, nb_iter=30, eps_iter=0.01, clip_min=0.0, clip_max=1.0, targeted=False) attackers.append(linf_PGD_attack) attackers.append(l2_PGD_attack) attackers.append(FGSM_attack) attackers.append(momentum_attack) log.info("Create model {} done!".format(model_name)) generate_and_save_adv_examples(datasetname, data_loader, attackers, save_dir_path)
def load_models(dataset): archs = [] model_path_list = [] if dataset == "CIFAR-10" or dataset == "CIFAR-100": for arch in ["resnet-110","WRN-28-10","WRN-34-10","resnext-8x64d","resnext-16x64d"]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, dataset, arch) if os.path.exists(test_model_path): archs.append(arch) model_path_list.append(test_model_path) else: log.info(test_model_path + " does not exist!") elif dataset == "TinyImageNet": # for arch in ["vgg11_bn","resnet18","vgg16_bn","resnext64_4","densenet121"]: for arch in MODELS_TEST_STANDARD[dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}@{}@*.pth.tar".format( PY_ROOT, dataset, arch) test_model_path = list(glob.glob(test_model_path))[0] if os.path.exists(test_model_path): archs.append(arch) model_path_list.append(test_model_path) else: log.info(test_model_path + "does not exist!") else: for arch in ["inceptionv3","inceptionv4", "inceptionresnetv2","resnet101", "resnet152"]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, dataset, arch) test_model_path = list(glob.glob(test_model_list_path)) if len(test_model_path) == 0: # this arch does not exists in args.dataset continue archs.append(arch) model_path_list.append(test_model_path[0]) models = [] print("begin construct model") if dataset == "TinyImageNet": for idx, arch in enumerate(archs): model = MetaLearnerModelBuilder.construct_tiny_imagenet_model(arch, dataset) model_path = model_path_list[idx] model.load_state_dict(torch.load(model_path, map_location=lambda storage, location: storage)["state_dict"]) model.cuda() model.eval() models.append(model) else: for arch in archs: model = StandardModel(dataset, arch, no_grad=True) model.cuda() model.eval() models.append(model) print("end construct model") return models
def main(): model = StandardModel(args.dataset, args.arch, no_grad=False, load_pretrained=False) model.cuda() model.train() device = torch.device("cuda") optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) model_path = '{}/train_pytorch_model/adversarial_train/TRADES/{}@{}@epoch_{}@batch_{}.pth.tar'.format( PY_ROOT, args.dataset, args.arch, args.epochs, args.batch_size) os.makedirs(os.path.dirname(model_path), exist_ok=True) print("After trained, the model will save to {}".format(model_path)) for epoch in range(1, args.epochs + 1): # adjust learning rate for SGD adjust_learning_rate(optimizer, epoch) # adversarial training train(args, model, device, train_loader, optimizer, epoch) # evaluation on natural examples print( '================================================================') eval_train(model, device, train_loader) eval_test(model, device, test_loader) print( '================================================================') # save checkpoint if epoch % args.save_freq == 0: state = { 'state_dict': model.state_dict(), 'epoch': epoch, 'optimizer': optimizer.state_dict() } torch.save(state, os.path.join(model_dir, model_path))
def __init__(self, dataset, arch, meta_batch_size, meta_step_size, inner_step_size, lr_decay_itr, num_inner_updates, num_support, num_query, epoch, load_task_mode, protocol, tot_num_tasks, data_loss_type, adv_norm, targeted, target_type, without_resnet): super(self.__class__, self).__init__() self.dataset = dataset self.meta_batch_size = meta_batch_size self.meta_step_size = meta_step_size self.inner_step_size = inner_step_size self.lr_decay_itr = lr_decay_itr self.epoch = epoch self.load_task_mode = load_task_mode self.targeted = targeted backbone = StandardModel(dataset, arch, no_grad=False, load_pretrained=False) self.network = MetaNetwork(backbone) self.network.cuda() trn_dataset = MetaTaskDataset(dataset, adv_norm, data_loss_type, tot_num_tasks, num_support, num_query, load_task_mode, protocol, targeted, target_type, without_resnet) self.train_loader = DataLoader(trn_dataset, batch_size=meta_batch_size, shuffle=True, num_workers=0, pin_memory=True) self.loss_fn = nn.MSELoss() self.fast_net = InnerLoop(self.network, num_inner_updates, self.inner_step_size, self.meta_batch_size, data_loss_type) # 并行执行每个task self.fast_net.cuda() self.opt = SGD(self.network.parameters(), lr=meta_step_size, momentum=0.9, weight_decay=0.0005)
if __name__ == "__main__": args = get_parse_args() if args.norm == "l2": args.epsilon = 4.6 elif args.norm == "linf": args.epsilon = 0.031372 if args.dataset == "ImageNet": args.epsilon = 0.05 data_loader = get_img_label_data_loader(args.dataset, args.batch_size, True) test_data_loader = get_img_label_data_loader(args.dataset, args.batch_size, False) os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) target_model = StandardModel(args.dataset, args.model, no_grad=True) target_model = target_model.cuda() current_state = State( (args.batch_size, IN_CHANNELS[args.dataset], IMAGE_SIZE[args.dataset][0], IMAGE_SIZE[args.dataset][1]), args.norm, args.epsilon) fcn = MyFCN((IN_CHANNELS[args.dataset], IMAGE_SIZE[args.dataset][0], IMAGE_SIZE[args.dataset][1]), args.n_actions) fcn.apply(fcn.init_weights) optimizer = Adam(fcn.parameters(), lr=args.learning_rate) agent = PixelWiseA3C(fcn, optimizer, args.episode_len, args.gamma) agent.model.cuda() agent.shared_model.cuda() i = 0 episode = 0 save_model_path = "{}/train_pytorch_model/sign_player/{}_untargeted_{}_attack_on_{}.pth.tar".format(
PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) surrogate_model = StandardModel(args.dataset, args.surrogate_arch, no_grad=False, load_pretrained=True) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch,
def main(): args = get_parse_args() if args.json_config: # If a json file is given, use the JSON file as the base, and then update it with args defaults = json.load(open(args.json_config))[args.dataset][args.norm] arg_vars = vars(args) arg_vars = {k: arg_vars[k] for k in arg_vars if arg_vars[k] is not None} defaults.update(arg_vars) args = SimpleNamespace(**defaults) args.exp_dir = os.path.join(args.exp_dir, get_exp_dir_name(args.dataset, args.norm, args.targeted, args.target_type, args)) os.makedirs(args.exp_dir, exist_ok=True) if args.test_archs: if args.attack_defense: log_file_path = os.path.join(args.exp_dir, 'run_defense_{}.log'.format(args.defense_model)) else: log_file_path = os.path.join(args.exp_dir, 'run.log') elif args.arch is not None: if args.attack_defense: log_file_path = os.path.join(args.exp_dir, 'run_defense_{}_{}.log'.format(args.arch, args.defense_model)) else: log_file_path = os.path.join(args.exp_dir, 'run_{}.log'.format(args.arch)) set_log_file(log_file_path) if args.attack_defense: assert args.defense_model is not None attacker = FrankWolfeBlackBoxAttack(args, args.dataset, args.targeted, args.target_type, args.epsilon, args.norm, args.sensing, args.grad_est, args.delta, 0, 1, max_queries=args.max_queries) torch.backends.cudnn.deterministic = True random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.test_archs: archs = [] if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format(arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format(arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() attacker.attack_all_images(args, arch, model, save_result_path) model.cpu()
from dataset.standard_model import StandardModel from torchstat import stat from thop import profile from thop import clever_format import torch import numpy as np models_CIFAR10 = { "pyramidnet272": StandardModel("CIFAR-10", "pyramidnet272", no_grad=True), "gdas": StandardModel("CIFAR-10", "gdas", no_grad=True), "WRN-28": StandardModel("CIFAR-10", "WRN-28-10-drop", no_grad=True), "WRN-40": StandardModel("CIFAR-10", "WRN-40-10-drop", no_grad=True), } models_CIFAR100 = { "pyramidnet272": StandardModel("CIFAR-100", "pyramidnet272", no_grad=True), "gdas": StandardModel("CIFAR-100", "gdas", no_grad=True), "WRN-28": StandardModel("CIFAR-100", "WRN-28-10-drop", no_grad=True), "WRN-40": StandardModel("CIFAR-100", "WRN-40-10-drop", no_grad=True), } models_tiny_imagenet = { "densenet-121": StandardModel("TinyImageNet", "densenet121", no_grad=True), "resnext32_4": StandardModel("TinyImageNet", "resnext32_4", no_grad=True), "resnext64_4": StandardModel("TinyImageNet", "resnext64_4", no_grad=True) } def params_count(model): """
def __init__(self, dataset, batch_size, meta_arch, meta_train_type, distill_loss, data_loss, norm, targeted, use_softmax, mode="meta"): if mode == "meta": target_str = "targeted_attack_random" if targeted else "untargeted_attack" # 2Q_DISTILLATION@CIFAR-100@TRAIN_I_TEST_II@model_resnet34@loss_pair_mse@dataloss_cw_l2_untargeted_attack@epoch_4@meta_batch_size_30@num_support_50@num_updates_12@lr_0.001@inner_lr_0.01.pth.tar self.meta_model_path = "{root}/train_pytorch_model/meta_simulator/{meta_train_type}@{dataset}@{split}@model_{meta_arch}@loss_{loss}@dataloss_{data_loss}_{norm}_{target_str}*inner_lr_0.01.pth.tar".format( root=PY_ROOT, meta_train_type=meta_train_type.upper(), dataset=dataset, split=SPLIT_DATA_PROTOCOL.TRAIN_I_TEST_II, meta_arch=meta_arch, loss=distill_loss, data_loss=data_loss, norm=norm, target_str=target_str) log.info("start using {}".format(self.meta_model_path)) self.meta_model_path = glob.glob(self.meta_model_path) pattern = re.compile(".*model_(.*?)@.*inner_lr_(.*?)\.pth.*") assert len(self.meta_model_path) > 0 self.meta_model_path = self.meta_model_path[0] log.info("load meta model {}".format(self.meta_model_path)) ma = pattern.match(os.path.basename(self.meta_model_path)) arch = ma.group(1) self.inner_lr = float(ma.group(2)) meta_backbone = self.construct_model(arch, dataset) self.meta_network = MetaNetwork(meta_backbone) self.pretrained_weights = torch.load(self.meta_model_path, map_location=lambda storage, location: storage) self.meta_network.load_state_dict(self.pretrained_weights["state_dict"]) log.info("Load model in epoch {}.".format(self.pretrained_weights["epoch"])) self.pretrained_weights = self.pretrained_weights["state_dict"] elif mode == "vanilla": target_str = "targeted" if targeted else "untargeted" arch = meta_arch # 2Q_DISTILLATION@CIFAR-100@TRAIN_I_TEST_II@model_resnet34@loss_pair_mse@dataloss_cw_l2_untargeted_attack@epoch_4@meta_batch_size_30@num_support_50@num_updates_12@lr_0.001@inner_lr_0.01.pth.tar self.meta_model_path = "{root}/train_pytorch_model/vanilla_simulator/{dataset}@{norm}_norm_{target_str}@{meta_arch}*.tar".format( root=PY_ROOT, dataset=dataset, meta_arch=meta_arch,norm=norm, target_str=target_str) log.info("start using {}".format(self.meta_model_path)) self.meta_model_path = glob.glob(self.meta_model_path) assert len(self.meta_model_path) > 0 self.meta_model_path = self.meta_model_path[0] log.info("load meta model {}".format(self.meta_model_path)) self.inner_lr = 0.01 self.meta_network = self.construct_model(meta_arch, dataset) self.pretrained_weights = torch.load(self.meta_model_path, map_location=lambda storage, location: storage) log.info("Load model in epoch {}.".format(self.pretrained_weights["epoch"])) self.pretrained_weights = self.pretrained_weights["state_dict"] elif mode == "deep_benign_images": arch = "resnet34" self.inner_lr = 0.01 self.meta_network = self.construct_model(arch, dataset) self.meta_model_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}@epoch_200@lr_0.1@batch_200.pth.tar".format( root=PY_ROOT, dataset=dataset, arch=arch) assert os.path.exists(self.meta_model_path), "{} does not exists!".format(self.meta_model_path) self.pretrained_weights = torch.load(self.meta_model_path, map_location=lambda storage, location: storage)[ "state_dict"] elif mode == "random_init": arch = "resnet34" self.inner_lr = 0.01 self.meta_network = self.construct_model(arch, dataset) self.pretrained_weights = self.meta_network.state_dict() elif mode == 'ensemble_avg': self.inner_lr = 0.01 self.archs = ["densenet-bc-100-12","resnet-110","vgg19_bn"] self.meta_network = [] # meta_network和pretrained_weights都改成list self.pretrained_weights = [] for arch in self.archs: model = StandardModel(dataset, arch, no_grad=False, load_pretrained=True) model.eval() model.cuda() self.meta_network.append(model) self.pretrained_weights.append(model.state_dict()) elif mode == "benign_images": self.inner_lr = 0.01 self.meta_model_path = "{root}/train_pytorch_model/meta_simulator_on_benign_images/{dataset}@{split}*@inner_lr_0.01.pth.tar".format( root=PY_ROOT, meta_train_type=meta_train_type.upper(), dataset=dataset, split=SPLIT_DATA_PROTOCOL.TRAIN_I_TEST_II) self.meta_model_path = glob.glob(self.meta_model_path) pattern = re.compile(".*model_(.*?)@.*") assert len(self.meta_model_path) > 0 self.meta_model_path = self.meta_model_path[0] ma = pattern.match(os.path.basename(self.meta_model_path)) log.info("Loading meta model from {}".format(self.meta_model_path)) arch = ma.group(1) self.pretrained_weights = torch.load(self.meta_model_path, map_location=lambda storage, location: storage)["state_dict"] meta_backbone = self.construct_model(arch, dataset) self.meta_network = MetaNetwork(meta_backbone) self.meta_network.load_state_dict(self.pretrained_weights) self.meta_network.eval() self.meta_network.cuda() elif mode == "reptile_on_benign_images": self.inner_lr = 0.01 self.meta_model_path = "{root}/train_pytorch_model/meta_simulator_reptile_on_benign_images/{dataset}@{split}*@inner_lr_0.01.pth.tar".format( root=PY_ROOT, meta_train_type=meta_train_type.upper(), dataset=dataset, split=SPLIT_DATA_PROTOCOL.TRAIN_I_TEST_II) self.meta_model_path = glob.glob(self.meta_model_path) pattern = re.compile(".*model_(.*?)@.*") assert len(self.meta_model_path) > 0 self.meta_model_path = self.meta_model_path[0] log.info("Loading meta model from {}".format(self.meta_model_path)) ma = pattern.match(os.path.basename(self.meta_model_path)) arch = ma.group(1) self.pretrained_weights = torch.load(self.meta_model_path, map_location=lambda storage, location: storage)["state_dict"] meta_backbone = self.construct_model(arch, dataset) self.meta_network = MetaNetwork(meta_backbone) self.meta_network.load_state_dict(self.pretrained_weights) self.meta_network.eval() self.meta_network.cuda() self.arch = arch self.dataset = dataset self.need_pair_distance = (distill_loss.lower()=="pair_mse") # self.need_pair_distance = False self.softmax = nn.Softmax(dim=1) self.mse_loss = nn.MSELoss(reduction="mean") self.pair_wise_distance = nn.PairwiseDistance(p=2) self.use_softmax = use_softmax if mode != "ensemble_avg": self.meta_network.load_state_dict(self.pretrained_weights) self.meta_network.eval() self.meta_network.cuda() self.batch_size = batch_size if mode == 'ensemble_avg': self.batch_weights = defaultdict(dict) for idx in range(len(self.pretrained_weights)): for i in range(batch_size): self.batch_weights[idx][i] = self.pretrained_weights[idx] else: self.batch_weights = dict() for i in range(batch_size): self.batch_weights[i] = self.pretrained_weights
transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) best_acc = 0 # best test accuracy if args.manualSeed is None: args.manualSeed = random.randint(1, 10000) random.seed(args.manualSeed) torch.manual_seed(args.manualSeed) torch.cuda.manual_seed_all(args.manualSeed) dataset = args.dataset if dataset != "ImageNet": for arch in os.listdir(args.dir_path): if not os.path.isdir(args.dir_path + "/" + arch): continue model = StandardModel(dataset, arch, True) model.cuda() model.eval() transform_test = transforms.Compose([ transforms.Resize(size=(IMAGE_SIZE[dataset][0], IMAGE_SIZE[dataset][1])), transforms.ToTensor(), ]) if dataset == "CIFAR-10": test_dataset = CIFAR10(IMAGE_DATA_ROOT[dataset], train=False, transform=transform_test) elif dataset == "CIFAR-100": test_dataset = CIFAR100(IMAGE_DATA_ROOT[dataset], train=False, transform=transform_test)
def __init__(self, tot_num_tasks, dataset, inner_batch_size, protocol): """ Args: num_samples_per_class: num samples to generate "per class" in one batch batch_size: size of meta batch size (e.g. number of functions) """ self.img_size = IMAGE_SIZE[dataset] self.dataset = dataset if protocol == SPLIT_DATA_PROTOCOL.TRAIN_I_TEST_II: self.model_names = MODELS_TRAIN_STANDARD[self.dataset] elif protocol == SPLIT_DATA_PROTOCOL.TRAIN_II_TEST_I: self.model_names = MODELS_TEST_STANDARD[self.dataset] elif protocol == SPLIT_DATA_PROTOCOL.TRAIN_ALL_TEST_ALL: self.model_names = MODELS_TRAIN_STANDARD[ self.dataset] + MODELS_TEST_STANDARD[self.dataset] self.model_dict = {} for arch in self.model_names: if StandardModel.check_arch(arch, dataset): model = StandardModel(dataset, arch, no_grad=False).eval() if dataset != "ImageNet": model = model.cuda() self.model_dict[arch] = model is_train = True preprocessor = DataLoaderMaker.get_preprocessor( IMAGE_SIZE[dataset], is_train) if dataset == "CIFAR-10": train_dataset = CIFAR10(IMAGE_DATA_ROOT[dataset], train=is_train, transform=preprocessor) elif dataset == "CIFAR-100": train_dataset = CIFAR100(IMAGE_DATA_ROOT[dataset], train=is_train, transform=preprocessor) elif dataset == "MNIST": train_dataset = MNIST(IMAGE_DATA_ROOT[dataset], train=is_train, transform=preprocessor) elif dataset == "FashionMNIST": train_dataset = FashionMNIST(IMAGE_DATA_ROOT[dataset], train=is_train, transform=preprocessor) elif dataset == "TinyImageNet": train_dataset = TinyImageNet(IMAGE_DATA_ROOT[dataset], preprocessor, train=is_train) elif dataset == "ImageNet": preprocessor = DataLoaderMaker.get_preprocessor( IMAGE_SIZE[dataset], is_train, center_crop=True) sub_folder = "/train" if is_train else "/validation" # Note that ImageNet uses pretrainedmodels.utils.TransformImage to apply transformation train_dataset = ImageFolder(IMAGE_DATA_ROOT[dataset] + sub_folder, transform=preprocessor) self.train_dataset = train_dataset self.total_num_images = len(train_dataset) self.all_tasks = dict() all_images_indexes = np.arange(self.total_num_images).tolist() for i in range(tot_num_tasks): self.all_tasks[i] = { "image": random.sample(all_images_indexes, inner_batch_size), "arch": random.choice(list(self.model_dict.keys())) }
log.info("Log file is located in {}".format(log_path)) log.info("All the data will be saved into {}".format(save_dir_path)) log.info("Using GPU {}".format(args.gpu)) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info('Called with args:') print_args(args) defaults = json.load(open(args.json_config))[args.dataset] arg_vars = vars(args) arg_vars = {k: arg_vars[k] for k in arg_vars if arg_vars[k] is not None} defaults.update(arg_vars) args = SimpleNamespace(**defaults) if args.norm == "linf": args.epsilon = defaults["linf_epsilon"] args.surrogate_arch = "resnet-110" if args.dataset.startswith( "CIFAR") else "resnet101" surrogate_model = StandardModel(args.dataset, args.surrogate_arch, False) trn_data_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, args.batch_size, is_train=True) # 生成的是训练集而非测试集 archs = [] for arch in MODELS_TRAIN_STANDARD[args.dataset]: if StandardModel.check_arch(arch, args.dataset): archs.append(arch) print("It will be use {} architectures".format(",".join(archs))) model_to_data = partition_dataset(archs, trn_data_loader, args.total_images) for arch in archs: model = StandardModel(args.dataset, arch, True) attacker = PriorRGFAttack(args.dataset, model, surrogate_model, args.targeted, args.target_type) log.info("Begin attack {}".format(arch)) with torch.no_grad():
with open("TotalList.txt", "a") as f: f.write(socket.gethostname() + ":" + args.results_folder + "\n") def onehot(ind, num_classes): vector = np.zeros([num_classes]) vector[ind] = 1 return vector.astype(np.float32) logger.info("build dataloader") train_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, args.batch_size, True) val_loader = DataLoaderMaker.get_img_label_data_loader(args.dataset, args.batch_size, False) model = StandardModel(args.dataset, args.arch, no_grad=False) model.cuda() model.train() def anneal_lr(epoch): if epoch < 100: return 1. elif epoch < 150: return 0.1 else: return 0.01 pgd_kwargs = { "eps": 16. / 255.,
else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format( osp.join(args.exp_dir, 'run.log'))) log.info('Called with args:') print_args(args) attacker = SimulateBanditsAttack(args, meta_finetuner) for arch in archs: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() attacker.attack_all_images(args, arch, model, save_result_path) model.cpu()
def main(): args = get_args() with open(args.config) as config_file: state = json.load(config_file)["attack"][args.targeted] state = SimpleNamespace(**state) if args.save_prefix is not None: state.save_prefix = args.save_prefix if args.arch is not None: state.arch = args.arch if args.test_archs is not None: state.test_archs = args.test_archs state.OSP = args.OSP state.targeted = args.targeted device = torch.device(args.device if torch.cuda.is_available() else "cpu") targeted_str = "untargeted" if not state.targeted else "targeted" if state.targeted: save_name = "{}/train_pytorch_model/TREMBA/{}_{}_generator.pth.tar".format( PY_ROOT, args.dataset, targeted_str) else: save_name = "{}/train_pytorch_model/TREMBA/{}_{}_generator.pth.tar".format( PY_ROOT, args.dataset, targeted_str) weight = torch.load(save_name, map_location=device)["state_dict"] data_loader = DataLoaderMaker.get_test_attacked_data( args.dataset, args.batch_size) encoder_weight = {} decoder_weight = {} for key, val in weight.items(): if key.startswith('0.'): encoder_weight[key[2:]] = val elif key.startswith('1.'): decoder_weight[key[2:]] = val archs = [] if args.test_archs: if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) args.arch = ",".join(archs) else: archs.append(args.arch) args.exp_dir = get_exp_dir_name(args.dataset, args.norm, args.targeted, args.target_type, args) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue if args.OSP: if state.source_model_name == "Adv_Denoise_Resnet152": source_model = resnet152_denoise() loaded_state_dict = torch.load((os.path.join( '{}/train_pytorch_model/TREMBA'.format(PY_ROOT), state.source_model_name + ".pth.tar"))) source_model.load_state_dict(loaded_state_dict) mean = np.array([0.485, 0.456, 0.406]) std = np.array([0.229, 0.224, 0.225]) # FIXME 仍然要改改 source_model = nn.Sequential(Normalize(mean, std), source_model) source_model.to(device) source_model.eval() if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.eval() encoder = ImagenetEncoder() decoder = ImagenetDecoder(args.dataset) encoder.load_state_dict(encoder_weight) decoder.load_state_dict(decoder_weight) model.to(device) encoder.to(device) encoder.eval() decoder.to(device) decoder.eval() F = Function(model, state.batch_size, state.margin, CLASS_NUM[args.dataset], state.targeted) total_success = 0 count_total = 0 queries = [] not_done = [] correct_all = [] for i, (images, labels) in enumerate(data_loader): images = images.to(device) labels = labels.to(device) logits = model(images) correct = torch.argmax(logits, dim=1).eq(labels).item() correct_all.append(int(correct)) if correct: if args.targeted: if args.target_type == 'random': target_labels = torch.randint( low=0, high=CLASS_NUM[args.dataset], size=labels.size()).long().cuda() invalid_target_index = target_labels.eq(labels) while invalid_target_index.sum().item() > 0: target_labels[ invalid_target_index] = torch.randint( low=0, high=logit.shape[1], size=target_labels[invalid_target_index]. shape).long().cuda() invalid_target_index = target_labels.eq(labels) elif args.target_type == 'least_likely': with torch.no_grad(): logit = model(images) target_labels = logit.argmin(dim=1) elif args.target_type == "increment": target_labels = torch.fmod( labels + 1, CLASS_NUM[args.dataset]).cuda() labels = target_labels[0].item() else: labels = labels[0].item() if args.OSP: hinge_loss = MarginLossSingle(state.white_box_margin, state.target) images.requires_grad = True latents = encoder(images) for k in range(state.white_box_iters): perturbations = decoder(latents) * state.epsilon logits = source_model( torch.clamp(images + perturbations, 0, 1)) loss = hinge_loss(logits, labels) grad = torch.autograd.grad(loss, latents)[0] latents = latents - state.white_box_lr * grad with torch.no_grad(): success, adv, query_count = EmbedBA( F, encoder, decoder, images[0], labels, state, latents.view(-1)) else: with torch.no_grad(): success, adv, query_count = EmbedBA( F, encoder, decoder, images[0], labels, state) not_done.append(1 - int(success)) total_success += int(success) count_total += int(correct) if success: queries.append(query_count) else: queries.append(args.max_queries) log.info( "image: {} eval_count: {} success: {} average_count: {} success_rate: {}" .format(i, F.current_counts, success, F.get_average(), float(total_success) / float(count_total))) F.new_counter() else: queries.append(0) not_done.append(1) log.info("The {}-th image is already classified incorrectly.". format(i)) correct_all = np.concatenate(correct_all, axis=0).astype(np.int32) query_all = np.array(queries).astype(np.int32) not_done_all = np.array(not_done).astype(np.int32) success = (1 - not_done_all) * correct_all success_query = success * query_all meta_info_dict = { "query_all": query_all.tolist(), "not_done_all": not_done_all.tolist(), "correct_all": correct_all.tolist(), "mean_query": np.mean(success_query[np.nonzero(success)[0]]).item(), "max_query": np.max(success_query[np.nonzero(success)[0]]).item(), "median_query": np.median(success_query[np.nonzero(success)[0]]).item(), "avg_not_done": np.mean(not_done_all[np.nonzero(correct_all)[0]].astype( np.float32)).item(), "args": vars(args) } with open(save_result_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) log.info("Done, write stats info to {}".format(save_result_path))
args.num_classes = CLASS_NUM[args.dataset] args.image_size = 64 device = 'cuda' if torch.cuda.is_available() else 'cpu' start_epoch = 0 model_path = '{}/train_pytorch_model/adversarial_train/feature_scatter/{}@{}@epoch_{}@batch_{}.pth.tar'.format( PY_ROOT, args.dataset, args.arch, args.max_epoch, args.batch_size_train) print("model will be saved to {}".format(model_path)) os.makedirs(os.path.dirname(model_path), exist_ok=True) # Data print('==> Preparing data..') train_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, args.batch_size_train, True) print('==> Building model..') basic_net = StandardModel(args.dataset, args.arch, no_grad=False).train().cuda() basic_net.apply(initialize_weights) def print_para(net): for name, param in net.named_parameters(): if param.requires_grad: print(name) print(param.data) break # config for feature scatter config_feature_scatter = { 'train': True, 'epsilon': 8.0 / 255 * 2,
with open(args.config) as config_file: state = json.load(config_file)["train"][targeted_str] state = SimpleNamespace(**state) state.targeted = args.targeted state.dataset = args.dataset state.batch_size = args.batch_size device = torch.device(args.gpu) train_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, state.batch_size, True) val_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, state.batch_size, False) nets = [] log.info("Initialize pretrained models.") for model_name in MODELS_TRAIN_STANDARD[args.dataset]: pretrained_model = StandardModel(args.dataset, model_name, no_grad=False) # pretrained_model.cuda() pretrained_model.eval() nets.append(pretrained_model) log.info("Initialize over!") model = nn.Sequential(ImagenetEncoder(), ImagenetDecoder(args.dataset)) model = model.cuda() optimizer_G = torch.optim.SGD(model.parameters(), state.learning_rate_G, momentum=state.momentum, weight_decay=0, nesterov=True) scheduler_G = torch.optim.lr_scheduler.StepLR(optimizer_G, step_size=state.epochs // state.schedule,
transform=train_preprocessor) elif dataset == "TinyImageNet": train_dataset = TinyImageNet(IMAGE_DATA_ROOT[dataset], train_preprocessor, train=True) batch_size = args.batch_size train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0) print('==> Building model..') arch_list = MODELS_TRAIN_STANDARD[args.dataset] model_dict = {} for arch in arch_list: if StandardModel.check_arch(arch, args.dataset): print("begin use arch {}".format(arch)) model = StandardModel(args.dataset, arch, no_grad=True) model_dict[arch] = model.eval() print("use arch {} done".format(arch)) print("==> Save gradient..") for arch, model in model_dict.items(): dump_path = "{}/benign_images_logits_pair/{}/{}_images.npy".format( PY_ROOT, args.dataset, arch) os.makedirs(os.path.dirname(dump_path), exist_ok=True) model = model.cuda() save_image_logits_pairs(model, train_loader, dump_path, args.batch_size, args.max_items) model.cpu()
assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) data_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, 1) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() attacker = ParsimoniousAttack(model, args) attack_all_images(data_loader, attacker, model, args, save_result_path) log.info("Attack {} in {} dataset done!".format(arch, args.dataset))
def main(): args = get_args_parse() os.environ[ "TORCH_HOME"] = "/home1/machen/meta_perturbations_black_box_attack/train_pytorch_model/real_image_model/ImageNet-pretrained" os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) target_str = "targeted" if args.targeted else "untargeted" json_conf = json.load(open( args.json_config))[args.dataset][target_str][args.norm] args = vars(args) args.update(json_conf) args = SimpleNamespace(**args) if args.targeted: if args.dataset == "ImageNet": args.max_queries = 50000 args.exp_dir = osp.join(args.exp_dir, get_exp_dir_name(args.dataset, args.surrogate_arch, args.norm, args.targeted, args.target_type, args)) # 随机产生一个目录用于实验 os.makedirs(args.exp_dir, exist_ok=True) if args.test_archs: if args.attack_defense: log_file_path = osp.join( args.exp_dir, 'run_defense_{}.log'.format(args.defense_model)) else: log_file_path = osp.join(args.exp_dir, 'run.log') elif args.arch is not None: if args.attack_defense: log_file_path = osp.join( args.exp_dir, 'run_defense_{}_{}.log'.format(args.arch, args.defense_model)) else: log_file_path = osp.join(args.exp_dir, 'run_{}.log'.format(args.arch)) set_log_file(log_file_path) archs = get_model_names(args) args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) layer = ['fc'] extractors = [] if args.surrogate_arch == "resnet50": resnet50 = models.resnet50(pretrained=True).eval() resnet50_extractor = ResNetFeatureExtractor(resnet50, layer).eval().cuda() extractors.append(resnet50_extractor) elif args.surrogate_arch == "resnet101": resnet101 = models.resnet101(pretrained=True).eval() resnet101_extractor = ResNetFeatureExtractor(resnet101, layer).eval().cuda() extractors.append(resnet101_extractor) elif args.surrogate_arch == "densenet121": densenet121 = models.densenet121(pretrained=True).eval() densenet121_extractor = DensenetFeatureExtractor(densenet121, layer).eval().cuda() extractors.append(densenet121_extractor) elif args.surrogate_arch == "densenet169": densenet169 = models.densenet169(pretrained=True).eval() densenet169_extractor = DensenetFeatureExtractor(densenet169, layer).eval().cuda() extractors.append(densenet169_extractor) directions_generator = TentativePerturbationGenerator(extractors, norm=args.norm, part_size=32, preprocess=True) attacker = VBADAttack(args, directions_generator) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() attacker.attack_all_images(args, arch, model, save_result_path) model.cpu()
def main(): parser = argparse.ArgumentParser( description='Square Attack Hyperparameters.') parser.add_argument('--norm', type=str, required=True, choices=['l2', 'linf']) parser.add_argument('--dataset', type=str, required=True) parser.add_argument('--exp-dir', default='logs', type=str, help='directory to save results and logs') parser.add_argument( '--gpu', type=str, required=True, help='GPU number. Multiple GPUs are possible for PT models.') parser.add_argument( '--p', type=float, default=0.05, help= 'Probability of changing a coordinate. Note: check the paper for the best values. ' 'Linf standard: 0.05, L2 standard: 0.1. But robust models require higher p.' ) parser.add_argument('--epsilon', type=float, help='Radius of the Lp ball.') parser.add_argument('--max_queries', type=int, default=10000) parser.add_argument( '--json-config', type=str, default= '/home1/machen/meta_perturbations_black_box_attack/configures/square_attack_conf.json', help='a configures file to be passed in instead of arguments') parser.add_argument('--batch_size', type=int, default=100) parser.add_argument('--targeted', action="store_true") parser.add_argument('--target_type', type=str, default='increment', choices=['random', 'least_likely', "increment"]) parser.add_argument('--attack_defense', action="store_true") parser.add_argument('--defense_model', type=str, default=None) parser.add_argument('--arch', default=None, type=str, help='network architecture') parser.add_argument('--test_archs', action="store_true") args = parser.parse_args() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.json_config: # If a json file is given, use the JSON file as the base, and then update it with args defaults = json.load(open(args.json_config))[args.dataset][args.norm] arg_vars = vars(args) arg_vars = { k: arg_vars[k] for k in arg_vars if arg_vars[k] is not None } defaults.update(arg_vars) args = SimpleNamespace(**defaults) if args.targeted and args.dataset == "ImageNet": args.max_queries = 50000 args.exp_dir = os.path.join( args.exp_dir, get_exp_dir_name(args.dataset, args.norm, args.targeted, args.target_type, args)) os.makedirs(args.exp_dir, exist_ok=True) if args.test_archs: if args.attack_defense: log_file_path = osp.join( args.exp_dir, 'run_defense_{}.log'.format(args.defense_model)) else: log_file_path = osp.join(args.exp_dir, 'run.log') elif args.arch is not None: if args.attack_defense: log_file_path = osp.join( args.exp_dir, 'run_defense_{}_{}.log'.format(args.arch, args.defense_model)) else: log_file_path = osp.join(args.exp_dir, 'run_{}.log'.format(args.arch)) set_log_file(log_file_path) if args.test_archs: archs = [] if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) attacker = MetaSimulatorSquareAttack(args.dataset, args.batch_size, args.targeted, args.target_type, args.epsilon, args.norm, max_queries=args.max_queries) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format( arch, args.dataset, save_result_path)) if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() attacker.attack_all_images(args, arch, model, save_result_path)
PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) surrogate_model = StandardModel(args.dataset, args.surrogate_arch, no_grad=False, load_pretrained=True) surrogate_model.cuda() surrogate_model.eval() attacker = NESRecitificationSurrogateGradientAttack( args, args.dataset, args.batch_size, args.targeted, args.target_type, args.epsilon, args.norm, 0.0, 1.0, args.max_queries) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format(
test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] args.arch = ", ".join(archs) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_file_path)) log.info('Called with args:') print_args(args) surrogate_model = StandardModel(args.dataset, args.surrogate_arch, False) surrogate_model.cuda() surrogate_model.eval() attacker = SwitchNeg(args.dataset, args.batch_size, args.targeted, args.target_type, args.epsilon, args.norm, 0.0, 1.0, args.max_queries) for arch in archs: if args.attack_defense: save_result_path = args.exp_dir + "/{}_{}_result.json".format( arch, args.defense_model) else: save_result_path = args.exp_dir + "/{}_result.json".format(arch) if os.path.exists(save_result_path): continue log.info("Begin attack {} on {}, result will be saved to {}".format(
def attack_all_images(self, args, arch, tmp_dump_path, result_dump_path): # subset_pos用于回调函数汇报汇总统计结果 model = StandardModel(args.dataset, arch, no_grad=True) model.cuda() model.eval() # 带有缩减功能的,攻击成功的图片自动删除掉 for data_idx, data_tuple in enumerate(self.dataset_loader): if os.path.exists(tmp_dump_path): with open(tmp_dump_path, "r") as file_obj: json_content = json.load(file_obj) resume_batch_idx = int(json_content["batch_idx"]) # resume for key in [ 'query_all', 'correct_all', 'not_done_all', 'success_all', 'success_query_all' ]: if key in json_content: setattr( self, key, torch.from_numpy(np.asarray( json_content[key])).float()) if data_idx < resume_batch_idx: # resume continue if args.dataset == "ImageNet": if model.input_size[-1] >= 299: images, true_labels = data_tuple[1], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[2] else: images, true_labels = data_tuple[0], data_tuple[1] if images.size(-1) != model.input_size[-1]: images = F.interpolate(images, size=model.input_size[-1], mode='bilinear', align_corners=True) # skip_batch_index_list = np.nonzero(np.asarray(chunk_skip_indexes[data_idx]))[0].tolist() selected = torch.arange( data_idx * args.batch_size, min((data_idx + 1) * args.batch_size, self.total_images)) # 选择这个batch的所有图片的index img_idx_to_batch_idx = ImageIdxToOrigBatchIdx(args.batch_size) images, true_labels = images.cuda(), true_labels.cuda() first_finetune = True finetune_queue = FinetuneQueue(args.batch_size, args.meta_seq_len, img_idx_to_batch_idx) prior_size = model.input_size[ -1] if not args.tiling else args.tile_size assert args.tiling == (args.dataset == "ImageNet") if args.tiling: upsampler = Upsample(size=(model.input_size[-2], model.input_size[-1])) else: upsampler = lambda x: x with torch.no_grad(): logit = model(images) pred = logit.argmax(dim=1) query = torch.zeros(images.size(0)).cuda() correct = pred.eq(true_labels).float() # shape = (batch_size,) not_done = correct.clone() # shape = (batch_size,) if args.targeted: if args.target_type == 'random': target_labels = torch.randint( low=0, high=CLASS_NUM[args.dataset], size=true_labels.size()).long().cuda() invalid_target_index = target_labels.eq(true_labels) while invalid_target_index.sum().item() > 0: target_labels[invalid_target_index] = torch.randint( low=0, high=logit.shape[1], size=target_labels[invalid_target_index].shape ).long().cuda() invalid_target_index = target_labels.eq(true_labels) elif args.target_type == 'least_likely': target_labels = logit.argmin(dim=1) elif args.target_type == "increment": target_labels = torch.fmod(true_labels + 1, CLASS_NUM[args.dataset]) else: raise NotImplementedError('Unknown target_type: {}'.format( args.target_type)) else: target_labels = None prior = torch.zeros(images.size(0), IN_CHANNELS[args.dataset], prior_size, prior_size).cuda() prior_step = self.gd_prior_step if args.norm == 'l2' else self.eg_prior_step image_step = self.l2_image_step if args.norm == 'l2' else self.linf_step proj_step = self.l2_proj_step if args.norm == 'l2' else self.linf_proj_step # 调用proj_maker返回的是一个函数 criterion = self.cw_loss if args.data_loss == "cw" else self.xent_loss adv_images = images.clone() for step_index in range(1, args.max_queries + 1): # Create noise for exporation, estimate the gradient, and take a PGD step dim = prior.nelement() / images.size( 0) # nelement() --> total number of elements exp_noise = args.exploration * torch.randn_like(prior) / ( dim**0.5 ) # parameterizes the exploration to be done around the prior exp_noise = exp_noise.cuda() q1 = upsampler( prior + exp_noise ) # 这就是Finite Difference算法, prior相当于论文里的v,这个prior也会更新,把梯度累积上去 q2 = upsampler( prior - exp_noise) # prior 相当于累积的更新量,用这个更新量,再去修改image,就会变得非常准 # Loss points for finite difference estimator q1_images = adv_images + args.fd_eta * q1 / self.norm(q1) q2_images = adv_images + args.fd_eta * q2 / self.norm(q2) predict_by_target_model = False if (step_index <= args.warm_up_steps or (step_index - args.warm_up_steps) % args.meta_predict_steps == 0): log.info("predict from target model") predict_by_target_model = True with torch.no_grad(): q1_logits = model(q1_images) q2_logits = model(q2_images) q1_logits = q1_logits / torch.norm( q1_logits, p=2, dim=-1, keepdim=True) # 加入normalize q2_logits = q2_logits / torch.norm( q2_logits, p=2, dim=-1, keepdim=True) finetune_queue.append(q1_images.detach(), q2_images.detach(), q1_logits.detach(), q2_logits.detach()) if step_index >= args.warm_up_steps: q1_images_seq, q2_images_seq, q1_logits_seq, q2_logits_seq = finetune_queue.stack_history_track( ) finetune_times = args.finetune_times if first_finetune else random.randint( 3, 5) # FIXME log.info("begin finetune for {} times".format( finetune_times)) self.meta_finetuner.finetune( q1_images_seq, q2_images_seq, q1_logits_seq, q2_logits_seq, finetune_times, first_finetune, img_idx_to_batch_idx) first_finetune = False else: with torch.no_grad(): q1_logits, q2_logits = self.meta_finetuner.predict( q1_images, q2_images, img_idx_to_batch_idx) q1_logits = q1_logits / torch.norm( q1_logits, p=2, dim=-1, keepdim=True) q2_logits = q2_logits / torch.norm( q2_logits, p=2, dim=-1, keepdim=True) l1 = criterion(q1_logits, true_labels, target_labels) l2 = criterion(q2_logits, true_labels, target_labels) # Finite differences estimate of directional derivative est_deriv = (l1 - l2) / (args.fd_eta * args.exploration ) # 方向导数 , l1和l2是loss # 2-query gradient estimate est_grad = est_deriv.view(-1, 1, 1, 1) * exp_noise # B, C, H, W, # Update the prior with the estimated gradient prior = prior_step( prior, est_grad, args.online_lr) # 注意,修正的是prior,这就是bandit算法的精髓 grad = upsampler(prior) # prior相当于梯度 ## Update the image: adv_images = image_step( adv_images, grad * correct.view(-1, 1, 1, 1), # 注意correct也是删减过的 args.image_lr) # prior放大后相当于累积的更新量,可以用来更新 adv_images = proj_step(images, args.epsilon, adv_images) adv_images = torch.clamp(adv_images, 0, 1) with torch.no_grad(): adv_logit = model(adv_images) # adv_pred = adv_logit.argmax(dim=1) adv_prob = F.softmax(adv_logit, dim=1) adv_loss = criterion(adv_logit, true_labels, target_labels) ## Continue query count if predict_by_target_model: query = query + 2 * not_done if args.targeted: not_done = not_done * ( 1 - adv_pred.eq(target_labels).float() ).float() # not_done初始化为 correct, shape = (batch_size,) else: not_done = not_done * adv_pred.eq( true_labels).float() # 只要是跟原始label相等的,就还需要query,还没有成功 success = (1 - not_done) * correct success_query = success * query not_done_loss = adv_loss * not_done not_done_prob = adv_prob[torch.arange(adv_images.size(0)), true_labels] * not_done log.info('Attacking image {} - {} / {}, step {}'.format( data_idx * args.batch_size, (data_idx + 1) * args.batch_size, self.total_images, step_index)) log.info(' not_done: {:.4f}'.format( len( np.where(not_done.detach().cpu().numpy().astype( np.int32) == 1)[0]) / float(args.batch_size))) log.info(' fd_scalar: {:.9f}'.format( (l1 - l2).mean().item())) if success.sum().item() > 0: log.info(' mean_query: {:.4f}'.format( success_query[success.byte()].mean().item())) log.info(' median_query: {:.4f}'.format( success_query[success.byte()].median().item())) if not_done.sum().item() > 0: log.info(' not_done_loss: {:.4f}'.format( not_done_loss[not_done.byte()].mean().item())) log.info(' not_done_prob: {:.4f}'.format( not_done_prob[not_done.byte()].mean().item())) not_done_np = not_done.detach().cpu().numpy().astype(np.int32) done_img_idx_list = np.where(not_done_np == 0)[0].tolist() delete_all = False if done_img_idx_list: for skip_index in done_img_idx_list: # 两次循环,第一次循环先汇报出去,第二次循环删除 batch_idx = img_idx_to_batch_idx[skip_index] pos = selected[batch_idx].item() # 先汇报被删减的值self.query_all for key in [ 'query', 'correct', 'not_done', 'success', 'success_query', 'not_done_loss', 'not_done_prob' ]: value_all = getattr(self, key + "_all") value = eval(key)[skip_index].item() value_all[pos] = value images, adv_images, prior, query, true_labels, target_labels, correct, not_done = \ self.delete_tensor_by_index_list(done_img_idx_list, images, adv_images, prior, query, true_labels, target_labels, correct, not_done) img_idx_to_batch_idx.del_by_index_list(done_img_idx_list) delete_all = images is None if delete_all: break # report to all stats the rest unsuccess for key in [ 'query', 'correct', 'not_done', 'success', 'success_query', 'not_done_loss', 'not_done_prob' ]: for img_idx, batch_idx in img_idx_to_batch_idx.proj_dict.items( ): pos = selected[batch_idx].item() value_all = getattr(self, key + "_all") value = eval(key)[img_idx].item() value_all[ pos] = value # 由于value_all是全部图片都放在一个数组里,当前batch选择出来 img_idx_to_batch_idx.proj_dict.clear() tmp_info_dict = { "batch_idx": data_idx + 1, "batch_size": args.batch_size } for key in [ 'query_all', 'correct_all', 'not_done_all', 'success_all', 'success_query_all' ]: value_all = getattr(self, key).detach().cpu().numpy().tolist() tmp_info_dict[key] = value_all with open(tmp_dump_path, "w") as result_file_obj: json.dump(tmp_info_dict, result_file_obj, sort_keys=True) log.info('Saving results to {}'.format(result_dump_path)) meta_info_dict = { "avg_correct": self.correct_all.mean().item(), "avg_not_done": self.not_done_all[self.correct_all.byte()].mean().item(), "mean_query": self.success_query_all[self.success_all.byte()].mean().item(), "median_query": self.success_query_all[self.success_all.byte()].median().item(), "max_query": self.success_query_all[self.success_all.byte()].max().item(), "correct_all": self.correct_all.detach().cpu().numpy().astype(np.int32).tolist(), "not_done_all": self.not_done_all.detach().cpu().numpy().astype(np.int32).tolist(), "query_all": self.query_all.detach().cpu().numpy().astype(np.int32).tolist(), "not_done_loss": self.not_done_loss_all[self.not_done_all.byte()].mean().item(), "not_done_prob": self.not_done_prob_all[self.not_done_all.byte()].mean().item(), "args": vars(args) } with open(result_dump_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) log.info("done, write stats info to {}".format(result_dump_path)) self.query_all.fill_(0) self.correct_all.fill_(0) self.not_done_all.fill_(0) self.success_all.fill_(0) self.success_query_all.fill_(0) self.not_done_loss_all.fill_(0) self.not_done_prob_all.fill_(0) model.cpu()
def main(): parser = argparse.ArgumentParser( description='Square Attack Hyperparameters.') parser.add_argument('--norm', type=str, required=True, choices=['l2', 'linf']) parser.add_argument('--dataset', type=str, required=True) parser.add_argument( '--gpu', type=str, required=True, help='GPU number. Multiple GPUs are possible for PT models.') parser.add_argument( '--p', type=float, default=0.05, help= 'Probability of changing a coordinate. Note: check the paper for the best values. ' 'Linf standard: 0.05, L2 standard: 0.1. But robust models require higher p.' ) parser.add_argument('--epsilon', type=float, help='Radius of the Lp ball.') parser.add_argument('--max_queries', type=int, default=1000) parser.add_argument( '--json-config', type=str, default= '/home1/machen/meta_perturbations_black_box_attack/configures/square_attack_conf.json', help='a configures file to be passed in instead of arguments') parser.add_argument('--batch_size', type=int, default=100) parser.add_argument('--targeted', action="store_true") parser.add_argument('--target_type', type=str, default='random', choices=['random', 'least_likely', "increment"]) parser.add_argument('--loss', type=str) args = parser.parse_args() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.json_config: # If a json file is given, use the JSON file as the base, and then update it with args defaults = json.load(open(args.json_config))[args.dataset][args.norm] arg_vars = vars(args) arg_vars = { k: arg_vars[k] for k in arg_vars if arg_vars[k] is not None } defaults.update(arg_vars) args = SimpleNamespace(**defaults) if args.targeted and args.dataset == "ImageNet": args.max_queries = 10000 save_dir_path = "{}/data_square_attack/{}/{}".format( PY_ROOT, args.dataset, "targeted_attack" if args.targeted else "untargeted_attack") os.makedirs(save_dir_path, exist_ok=True) loss_type = "cw" if not args.targeted else "xent" args.loss = loss_type log_path = osp.join( save_dir_path, get_log_path(args.dataset, loss_type, args.norm, args.targeted, args.target_type)) set_log_file(log_path) log.info('Command line is: {}'.format(' '.join(sys.argv))) log.info("Log file is written in {}".format(log_path)) log.info('Called with args:') print_args(args) trn_data_loader = DataLoaderMaker.get_img_label_data_loader( args.dataset, args.batch_size, is_train=True) models = [] for arch in MODELS_TRAIN_STANDARD[args.dataset]: if StandardModel.check_arch(arch, args.dataset): model = StandardModel(args.dataset, arch, True) model = model.eval() models.append({"arch_name": arch, "model": model}) model_data_dict = defaultdict(list) for images, labels in trn_data_loader: model_info = random.choice(models) arch = model_info["arch_name"] model = model_info["model"] if images.size(-1) != model.input_size[-1]: images = F.interpolate(images, size=model.input_size[-1], mode='bilinear', align_corners=True) model_data_dict[(arch, model)].append((images, labels)) log.info("Assign data to multiple models over!") attacker = SquareAttack(args.dataset, args.targeted, args.target_type, args.epsilon, args.norm, max_queries=args.max_queries) attacker.attack_all_images(args, model_data_dict, save_dir_path) log.info("All done!")
def main(args, result_dir_path): log.info('Loading %s model and test data' % args.dataset) dataset_loader = DataLoaderMaker.get_test_attacked_data(args.dataset, 1) if args.test_archs: archs = [] if args.dataset == "CIFAR-10" or args.dataset == "CIFAR-100": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/{}/checkpoint.pth.tar".format( PY_ROOT, args.dataset, arch) if os.path.exists(test_model_path): archs.append(arch) else: log.info(test_model_path + " does not exists!") elif args.dataset == "TinyImageNet": for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{root}/train_pytorch_model/real_image_model/{dataset}@{arch}*.pth.tar".format( root=PY_ROOT, dataset=args.dataset, arch=arch) test_model_path = list(glob.glob(test_model_list_path)) if test_model_path and os.path.exists(test_model_path[0]): archs.append(arch) else: for arch in MODELS_TEST_STANDARD[args.dataset]: test_model_list_path = "{}/train_pytorch_model/real_image_model/{}-pretrained/checkpoints/{}*.pth".format( PY_ROOT, args.dataset, arch) test_model_list_path = list(glob.glob(test_model_list_path)) if len(test_model_list_path ) == 0: # this arch does not exists in args.dataset continue archs.append(arch) else: assert args.arch is not None archs = [args.arch] if args.attack_defense: meta_model_path = '{}/train_pytorch_model/meta_grad_regression/{}_without_resnet.pth.tar'.format( PY_ROOT, args.dataset) else: meta_model_path = '{}/train_pytorch_model/meta_grad_regression/{}.pth.tar'.format( PY_ROOT, args.dataset) assert os.path.exists(meta_model_path), "{} does not exist!".format( meta_model_path) meta_model = load_meta_model(meta_model_path) log.info("Load meta model from {}".format(meta_model_path)) attack = MetaGradAttack(args, norm=args.norm, epsilon=args.epsilon, targeted=args.targeted, search_steps=args.binary_steps, max_steps=args.maxiter, use_log=not args.use_zvalue, cuda=not args.no_cuda) for arch in archs: if args.attack_defense: model = DefensiveModel(args.dataset, arch, no_grad=True, defense_model=args.defense_model) else: model = StandardModel(args.dataset, arch, no_grad=True) model.cuda().eval() query_all = [] not_done_all = [] correct_all = [] img_no = 0 total_success = 0 l2_total = 0.0 avg_step = 0 avg_time = 0 avg_qry = 0 if args.attack_defense: result_dump_path = result_dir_path + "/{}_{}_result.json".format( arch, args.defense_model) else: result_dump_path = result_dir_path + "/{}_result.json".format(arch) # if os.path.exists(result_dump_path): # continue log.info("Begin attack {} on {}".format(arch, args.dataset)) for i, data_tuple in enumerate(dataset_loader): if args.dataset == "ImageNet": if model.input_size[-1] >= 299: img, true_labels = data_tuple[1], data_tuple[2] else: img, true_labels = data_tuple[0], data_tuple[2] else: img, true_labels = data_tuple[0], data_tuple[1] args.init_size = model.input_size[-1] if img.size(-1) != model.input_size[-1]: img = F.interpolate(img, size=model.input_size[-1], mode='bilinear', align_corners=True) img, true_labels = img.to(0), true_labels.to(0) with torch.no_grad(): pred_logit = model(img) pred_label = pred_logit.argmax(dim=1) correct = pred_label.eq(true_labels).detach().cpu().numpy().astype( np.int32) correct_all.append(correct) if pred_label[0].item() != true_labels[0].item(): log.info( "Skip wrongly classified image no. %d, original class %d, classified as %d" % (i, pred_label.item(), true_labels.item())) query_all.append(0) not_done_all.append( 1) # 原本就分类错误,not_done给1,假如99%都原本分类错误的话, avg_not_done = 99% continue img_no += 1 timestart = time.time() meta_model_copy = copy.deepcopy(meta_model) if args.targeted: if args.target_type == 'random': target_labels = torch.randint( low=0, high=CLASS_NUM[args.dataset], size=true_labels.size()).long().cuda() invalid_target_index = target_labels.eq(true_labels) while invalid_target_index.sum().item() > 0: target_labels[invalid_target_index] = torch.randint( low=0, high=pred_logit.shape[1], size=target_labels[invalid_target_index].shape ).long().cuda() invalid_target_index = target_labels.eq(true_labels) elif args.target_type == 'least_likely': target_labels = pred_logit.argmin(dim=1) elif args.target_type == "increment": target_labels = torch.fmod(true_labels + 1, CLASS_NUM[args.dataset]) else: raise NotImplementedError('Unknown target_type: {}'.format( args.target_type)) else: target_labels = None target = true_labels if not args.targeted else target_labels adv, const, first_step, success_queries = attack.run( model, meta_model_copy, img, target) timeend = time.time() if len(adv.shape) == 3: adv = adv.reshape((1, ) + adv.shape) adv = torch.from_numpy(adv).permute(0, 3, 1, 2).cuda() # BHWC -> BCHW diff = (adv - img).detach().cpu().numpy() l2_distortion = np.sqrt(np.sum(np.square(diff))).item() with torch.no_grad(): adv_pred_logit = model(adv) adv_pred_label = adv_pred_logit.argmax(dim=1) success = False if not args.targeted: # target is true label if adv_pred_label[0].item() != target[0].item(): success = True else: if adv_pred_label[0].item() == target[0].item(): success = True if success_queries > args.max_queries: success = False if success: # (first_step-1)//args.finetune_intervalargs.update_pixels2+first_step # The first step is the iterations used that find the adversarial examples; # args.finetune_interval is the finetuning per iterations; # args.update_pixels is the queried pixels each iteration. # Currently, we find only f(x+h)-f(x) could estimate the gradient well, so args.update_pixels*1 in my updated codes. not_done_all.append(0) # only 1 query for i pixle, because the estimated function is f(x+h)-f(x)/h query_all.append(success_queries) total_success += 1 l2_total += l2_distortion avg_step += first_step avg_time += timeend - timestart # avg_qry += (first_step-1)//args.finetune_interval*args.update_pixels*1+first_step avg_qry += success_queries log.info("Attack {}-th image: {}, query:{}".format( i, "success", success_queries)) else: not_done_all.append(1) query_all.append(args.max_queries) log.info("Attack {}-th image: {}, query:{}".format( i, "fail", success_queries)) model.cpu() if total_success != 0: log.info( "[STATS][L1] total = {}, time = {:.3f}, distortion = {:.5f}, avg_step = {:.5f},avg_query = {:.5f}, success_rate = {:.3f}" .format(img_no, avg_time / total_success, l2_total / total_success, avg_step / total_success, avg_qry / total_success, total_success / float(img_no))) correct_all = np.concatenate(correct_all, axis=0).astype(np.int32) query_all = np.array(query_all).astype(np.int32) not_done_all = np.array(not_done_all).astype(np.int32) success = (1 - not_done_all) * correct_all success_query = success * query_all # query_all_bounded = query_all.copy() # not_done_all_bounded = not_done_all.copy() # out_of_bound_indexes = np.where(query_all_bounded > args.max_queries)[0] # if len(out_of_bound_indexes) > 0: # not_done_all_bounded[out_of_bound_indexes] = 1 # success_bounded = (1-not_done_all_bounded) * correct_all # success_query_bounded = success_bounded * query_all_bounded # # query_threshold_success_rate_bounded, query_success_rate_bounded = success_rate_and_query_coorelation(query_all_bounded, not_done_all_bounded) # success_rate_to_avg_query_bounded = success_rate_avg_query(query_all_bounded, not_done_all_bounded) meta_info_dict = { "query_all": query_all.tolist(), "not_done_all": not_done_all.tolist(), "correct_all": correct_all.tolist(), "mean_query": np.mean(success_query[np.nonzero(success)[0]]).item(), "max_query": np.max(success_query[np.nonzero(success)[0]]).item(), "median_query": np.median(success_query[np.nonzero(success)[0]]).item(), "avg_not_done": np.mean( not_done_all.astype( np.float32)[np.nonzero(correct_all)[0]]).item(), # "mean_query_bounded_max_queries": np.mean(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "max_query_bounded_max_queries": np.max(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "median_query_bounded_max_queries": np.median(success_query_bounded[np.nonzero(success_bounded)[0]]).item(), # "avg_not_done_bounded_max_queries": np.mean(not_done_all_bounded.astype(np.float32)).item(), # "query_threshold_success_rate_dict_bounded": query_threshold_success_rate_bounded, # "query_success_rate_dict_bounded": query_success_rate_bounded, # "success_rate_to_avg_query_bounded": success_rate_to_avg_query_bounded, "args": vars(args) } with open(result_dump_path, "w") as result_file_obj: json.dump(meta_info_dict, result_file_obj, sort_keys=True) log.info("done, write stats info to {}".format(result_dump_path))