def __init__(self, dataset, model): super(CNNCertBase, self).__init__(dataset, model) self.num_classes = get_num_classes(dataset) input_shape = get_input_shape(dataset) new_input_shape = (input_shape[1], input_shape[2], input_shape[0]) self.k_model = sequential_torch2keras(model_transform(self.model, input_shape), dataset) global graph global sess with sess.as_default(): with graph.as_default(): # Save the transformed Keras model to a temporary place so that the tool can read from file # The tool can only init model from file... sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True) self.k_model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) self.k_model.save('tmp/tmp.pt') self.new_model = nl.CNNModel('tmp/tmp.pt', new_input_shape) self.weights = self.new_model.weights self.biases = self.new_model.biases # print(self.new_model.summary()) try: check_consistency(self.model, self.k_model, input_shape) except Exception: raise Exception("Somehow the transformed model behaves differently from the original model.") self.LP = False self.LPFULL = False self.method = "ours" self.dual = False
def verify(self, input, label, norm_type, radius): """ Here we overwrite the base class verify() method """ # only support Linfty norm assert norm_type == 'inf' # firstly check the clean prediction input = self.input_preprocess(input) xs = input.unsqueeze(0) clean_preds = self.model(xs.cuda()).detach().cpu().numpy() clean_pred = np.argmax(clean_preds[0]) if clean_pred != label: return False input = torch.Tensor(input) m_radius = radius / self.coef self.bound.calculate_bound(input, m_radius) for i in range(get_num_classes(self.dataset)): if i != label: ok = self.bound.verify(label, i) if not ok: return False return True
def main(): args = Args().get_args() kwargs = vars(args) checkpoint = torch.load(args.checkpoint) base_classifier = get_architecture(checkpoint["arch"], args.dataset) base_classifier.load_state_dict(checkpoint['state_dict']) attacker = SmoothAttack(base_classifier) smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.sigma) dataset = get_dataset(args.dataset, 'test') average_nat = [] average_adv = [] j_header('index', 'nat_y', 'adv_y', 'nat_rad', 'adv_rad', 'success') figure = FigureSaver() for i in range(0, len(dataset), args.skip): (x, label) = dataset[i] x = x.cuda() first_x = x.data nat_pred, nat_rad = smoothed_classifier.certify( x, args.N0, args.N, args.alpha, args.batch) if nat_pred is -1: continue if args.dataset == DATASETS[0]: # ImageNet targets = [j for j in range(0, 1000, 100) if j is not label] else: targets = [j for j in range(10) if j is not label] best_rad = -10.0 best_image = None best_target = -1 for target in targets: adv_x = attacker.perturb(x=first_x, y=target, **kwargs) # If you want to do wasserstein attack, uncomment the following and change the attacker to wasserstein # adv_x = attacker.perturb(x=first_x, y=target, eps=args.sigma, steps=args.steps, batch=args.batch) adv_pred, adv_rad = smoothed_classifier.certify( adv_x, args.N0, 2 * args.N0, args.alpha, args.batch) adv_suc = (adv_pred != label) and (adv_pred != -1) and (nat_pred != -1) adv_rad = adv_rad if adv_suc else -adv_rad if adv_rad > best_rad: best_rad = adv_rad best_image = adv_x.data best_target = target figure.save(best_image, i, 'best={}'.format(best_target)) figure.save(first_x, i, 'natural') best_pred, best_rad = smoothed_classifier.certify( best_image, args.N0, args.N, args.alpha, args.batch) j_print(i, label, best_target, nat_rad, best_rad) average_adv.append(best_rad) average_nat.append(nat_rad) average_nat = np.array(average_nat) average_adv = np.array(average_adv) print('Average nat radii {}, Average adv radii {}'.format( average_nat.mean(), average_adv.mean()))
def verify(self, input, label, norm_type, radius): # only support Linfty norm assert norm_type == 'inf' if self.new_model is None: # init at the first time before = time() print(f"Init model for {self.__class__.__name__}...") in_shape = list(input.shape) self.build_new_model(input) self.prepare_solver(in_shape) after = time() print( "Init done, time", str(datetime.timedelta(seconds=(after - before))), ) # firstly check the clean prediction input = self.input_preprocess(input) xs = input.unsqueeze(0) clean_preds = self.model(xs.cuda()).detach().cpu().numpy() clean_pred = np.argmax(clean_preds[0]) if clean_pred != label: return False m_radius = radius / self.coef input = input.contiguous().view(-1) self.bound.calculate_bound(input, m_radius) for i in range(get_num_classes(self.dataset)): if i != label: ok = self.bound.verify(label, i) if not ok: return False return True
def init(dataset, filename, epsilon, batch_size): global myclient D_in = datasets.get_num_features(dataset) D_out = datasets.get_num_classes(dataset) nParams = datasets.get_num_params(dataset) model = SoftmaxModel(D_in, D_out) train_cut = 0.8 myclient = client.Client(dataset, filename, batch_size, model, train_cut) global samples samples = [] global this_batch_size this_batch_size = batch_size def lnprob(x, alpha): return -(alpha / 2) * np.linalg.norm(x) if epsilon > 0: if diffPriv13: nwalkers = max(4 * nParams, 250) sampler = emcee.EnsembleSampler(nwalkers, nParams, lnprob, args=[epsilon]) p0 = [np.random.rand(nParams) for i in range(nwalkers)] pos, _, state = sampler.run_mcmc(p0, 100) sampler.reset() sampler.run_mcmc(pos, 1000, rstate0=state) print("Mean acceptance fraction:", np.mean(sampler.acceptance_fraction)) samples = sampler.flatchain else: sigma = np.sqrt(2 * np.log(1.25)) / epsilon noise = sigma * np.random.randn(batch_size, expected_iters, nParams) samples = np.sum(noise, axis=0) else: samples = np.zeros((expected_iters, nParams)) return nParams
def verify(self, input, label, norm_type, radius): """ Here we overwrite the base class verify() method """ # only support Linfty norm assert norm_type == 'inf' if self.new_model is None: # init at the first time before = time() print(f"Init model for {self.__class__.__name__}...") in_shape = list(input.shape) self.build_new_model(input) self.prepare_solver(in_shape) after = time() print( "Init done, time", str(datetime.timedelta(seconds=(after - before))), ) # firstly check the clean prediction input = self.input_preprocess(input) xs = input.unsqueeze(0) clean_preds = self.model(xs.cuda()).detach().cpu().numpy() clean_pred = np.argmax(clean_preds[0]) if clean_pred != label: return False m_radius = radius / self.coef input = input.contiguous().view(-1) self.prebound.calculate_bound(input, m_radius) self.bound.construct(self.prebound.l, self.prebound.u, input, m_radius) for i in range(get_num_classes(self.dataset)): if i != label: self.bound.prepare_verify(label, i) try: # self.bound.prob.solve(verbose=True) # model.setParam(GRB.Param.TimeLimit, timeout) self.bound.prob.solve(solver=cp.GUROBI, verbose=False, BestObjStop=-1e-6, TimeLimit=70, Threads=20) except: return False print(self.bound.prob.status, self.bound.prob.value, file=sys.stderr) if self.bound.prob.status not in [ 'optimal' ] or self.bound.prob.value < 0.: return False return True
def verify(self, input, label, norm_type, radius): """ Here we overwrite the base class verify() method """ in_shape = list(input.shape) # only support Linfty norm assert norm_type == 'inf' if self.new_model is None: # init at the first time before = time() print(f"Init model for {self.__class__.__name__}...") self.build_new_model(input) self.prepare_solver(in_shape) after = time() print( "Init done, time", str(datetime.timedelta(seconds=(after - before))), ) # firstly check the clean prediction input = self.input_preprocess(input) xs = input.unsqueeze(0) clean_preds = self.model(xs.cuda()).detach().cpu().numpy() clean_pred = np.argmax(clean_preds[0]) if clean_pred != label: return False m_radius = radius / self.coef input = input.contiguous().view(-1) self.prebound.calculate_bound(input, m_radius) bl = [ np.maximum(self.prebound.l[i], 0) if i > 0 else self.prebound.l[i] for i in range(len(self.prebound.l)) ] bu = [ np.maximum(self.prebound.u[i], 0) if i > 0 else self.prebound.u[i] for i in range(len(self.prebound.u)) ] pv = BaselinePointVerifierExt(self.new_model, in_shape, self.in_min, self.in_max) for i in range(get_num_classes(self.dataset)): if i != label: pv.create_cmat(input, label, i, m_radius, bl, bu) pv.run() if pv.prob.status not in ['unbounded', 'unbounded_inaccurate' ] and pv.prob.value > 0.: return False return True
def __init__(self, dataset, model): super(CNNCertBase, self).__init__(dataset, model) self.num_classes = get_num_classes(dataset) self.activation = list() for layer in self.model: if isinstance(layer, nn.ReLU): self.activation.append('ada') elif isinstance(layer, nn.Sigmoid): self.activation.append('sigmoid') elif isinstance(layer, nn.Tanh): self.activation.append('tanh') # actually there is another activation called arctan, # but there is no corresponding one in pytorch so we ignore it self.activation = list(set(self.activation)) assert len(self.activation) == 1 self.activation = self.activation[0] input_shape = get_input_shape(dataset) new_input_shape = (input_shape[1], input_shape[2], input_shape[0]) self.k_model = sequential_torch2keras(self.model, dataset) global graph global sess with sess.as_default(): with graph.as_default(): print(self.k_model.summary()) try: assert check_consistency(self.model, self.k_model, input_shape) == True except: raise Exception("Somehow the transformed model behaves differently from the original model.") self.new_model = Model(self.k_model, new_input_shape) # Set correct linear_bounds function self.linear_bounds = None if self.activation == 'relu': self.linear_bounds = relu_linear_bounds elif self.activation == 'ada': self.linear_bounds = ada_linear_bounds elif self.activation == 'sigmoid': self.linear_bounds = sigmoid_linear_bounds elif self.activation == 'tanh': self.linear_bounds = tanh_linear_bounds elif self.activation == 'arctan': self.linear_bounds = atan_linear_bounds
def __init__(self, args, **kwargs): super(DecTrainer, self).__init__(args, **kwargs) # dataloader self.trainloader = get_dataloader(args, cfg, 'train') # self.trainloader_val = get_dataloader(args, cfg, 'train_voc') self.valloader = get_dataloader(args, cfg, 'val') self.denorm = self.trainloader.dataset.denorm self.use_triplet = args.use_triplet self.loss_3d = args.loss_3d self.normalize_feature = args.normalize_feature self.nclass = get_num_classes(args) self.classNames = get_class_names(args) assert self.nclass == len(self.classNames) - 1 self.classIndex = {} for i, cname in enumerate(self.classNames): self.classIndex[cname] = i # model self.enc = get_model(cfg.NET, num_classes=self.nclass) self.criterion_cls = get_criterion(cfg.NET.LOSS) # optimizer using different LR enc_params = self.enc.parameter_groups(cfg.NET.LR, cfg.NET.WEIGHT_DECAY) self.optim_enc = self.get_optim(enc_params, cfg.NET) # checkpoint management self._define_checkpoint('enc', self.enc, self.optim_enc) self._load_checkpoint(args.resume) self.fixed_batch = None self.fixed_batch_path = args.fixed_batch_path if os.path.isfile(self.fixed_batch_path): print("Loading fixed batch from {}".format(self.fixed_batch_path)) self.fixed_batch = torch.load(self.fixed_batch_path) # using cuda if cfg.NUM_GPUS != 0: self.enc = nn.DataParallel(self.enc) self.criterion_cls = nn.DataParallel(self.criterion_cls) self.enc = self.enc.cuda() self.criterion_cls = self.criterion_cls.cuda() # CHANGE: visual self.visual_times = 0 self.dataset = args.dataset.lower()
def verify(self, input, label, norm_type, radius): assert norm_type == 'inf' self.load_config() input = self.input_preprocess(input) m_radius = radius / self.coef image = input.permute(1, 2, 0).contiguous().numpy().reshape(-1) specLB = np.copy(image) specUB = np.copy(image) domain = config.domain pred, nn, nlb, nub = self.eran.analyze_box( specLB, specUB, init_domain(domain), config.timeout_lp, config.timeout_milp, config.use_default_heuristic) if pred == label: specLB = np.clip(image - m_radius, self.in_min, self.in_max) specUB = np.clip(image + m_radius, self.in_min, self.in_max) perturbed_label, _, nlb, nub = self.eran.analyze_box( specLB, specUB, domain, config.timeout_lp, config.timeout_milp, config.use_default_heuristic) # print("nlb ", nlb[len(nlb) - 1], " nub ", nub[len(nub) - 1]) if (perturbed_label == label): return True else: if config.complete: # print("complete verification here") constraints = get_constraints_for_dominant_label( label, get_num_classes(self.dataset)) verified_flag, adv_image = verify_network_with_milp( nn, specLB, specUB, nlb, nub, constraints) if (verified_flag == True): return True else: return False else: return False else: raise Exception( "Wrong prediction... This should not happen according to evaluate.py" ) # print('Prediction is wrong') return False
def train(loader: DataLoader, model: torch.nn.Module, optimizer: Optimizer, epoch: int, noise_sd: float, device: torch.device, writer=None): # switch to train mode model.train() lbd = args.lbd if args.deferred and epoch <= args.lr_step_size: lbd = 0 cl, rl = macer_train(sigma=noise_sd, lbd=lbd, gauss_num=args.num_noise_vec, beta=args.beta, gamma=args.margin, num_classes=get_num_classes(args.dataset), model=model, trainloader=loader, optimizer=optimizer, device=device) writer.add_scalar('loss/train', cl, epoch) writer.add_scalar('loss/robust', rl, epoch) return cl
def __init__(self, dataset, filename, train_cut=.80): # initializes dataset self.batch_size = 4 Dataset = datasets.get_dataset(dataset) transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) self.trainset = Dataset(filename, "data/" + dataset, is_train=True, train_cut=train_cut, transform=transform) self.testset = Dataset(filename, "data/" + dataset, is_train=False, train_cut=train_cut, transform=transform) self.trainloader = torch.utils.data.DataLoader( self.trainset, batch_size=self.batch_size, shuffle=True) self.testloader = torch.utils.data.DataLoader(self.testset, batch_size=len( self.testset), shuffle=False) D_in = datasets.get_num_features(dataset) D_out = datasets.get_num_classes(dataset) self.model = SoftmaxModel(D_in, D_out) # self.model = MNISTCNNModel() # self.model = LFWCNNModel() # self.model = SVMModel(D_in, D_out) # self.criterion = nn.MultiLabelMarginLoss() ### Tunables ### self.criterion = nn.CrossEntropyLoss() self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001) self.aggregatedGradients = [] self.loss = 0.0
def __init__(self, args, **kwargs): super(DecTrainer, self).__init__(args, **kwargs) # dataloader self.trainloader = get_dataloader(args, cfg, 'train') self.trainloader_val = get_dataloader(args, cfg, 'train_voc') self.valloader = get_dataloader(args, cfg, 'val') self.denorm = self.trainloader.dataset.denorm self.nclass = get_num_classes(args) self.classNames = get_class_names(args)[:-1] assert self.nclass == len(self.classNames) self.classIndex = {} for i, cname in enumerate(self.classNames): self.classIndex[cname] = i # model self.enc = get_model(cfg.GENERATOR, num_classes=self.nclass) self.criterion_cls = get_criterion(cfg.GENERATOR.LOSS) print(self.enc) # optimizer using different LR enc_params = self.enc.parameter_groups(cfg.GENERATOR.LR, cfg.GENERATOR.WEIGHT_DECAY) self.optim_enc = self.get_optim(enc_params, cfg.GENERATOR) # checkpoint management self._define_checkpoint('enc', self.enc, self.optim_enc) self._load_checkpoint(args.resume) self.fixed_batch = None self.fixed_batch_path = args.fixed_batch_path if os.path.isfile(self.fixed_batch_path): print("Loading fixed batch from {}".format(self.fixed_batch_path)) self.fixed_batch = torch.load(self.fixed_batch_path) # using cuda self.enc = nn.DataParallel(self.enc).cuda() self.criterion_cls = nn.DataParallel(self.criterion_cls).cuda()
def main(batchsize, epsilon, dataset, data_filename, dataclass): train_cut = 0.8 iter_time = 1 clients = [] average_loss = [] test_accuracy_rate = [] D_in = datasets.get_num_features(dataset) D_out = datasets.get_num_classes(dataset) global batch_size batch_size = batchsize global nParams nParams = datasets.get_num_params(dataset) print("Creating clients") # for i in range(10): # model = returnModel(D_in, D_out) # clients.append(Client("lfw", "lfw_maleness_train" + str(i), batch_size, model, train_cut)) model = returnModel(D_in, D_out) # clients.append(Client("lfw", "lfw_maleness_person61_over20", batch_size, model, train_cut)) clients.append(Client(dataset, data_filename, batch_size, model, train_cut)) print("Training for iterations") for iter in range(iter_time): # Calculate and aggregate gradients for i in range(1): grad = clients[i].getGrad() if epsilon == 0: showImage(grad, dataset, dataclass, batch_size, 0) else: varyEpsilonShowImage(grad, batch_size, epsilon, dataset, data_class)
def calc_radius(self, input, label, norm_type, upper=0.5, eps=1e-2): # only support L2 norm assert norm_type == '2' xs = input.unsqueeze(0) clean_preds = self.model(xs.cuda()).detach().cpu().numpy() clean_pred = np.argmax(clean_preds[0]) if clean_pred != label: return 0. x_op = tf.placeholder(tf.float32, shape=( None, input.shape[0], input.shape[1], input.shape[2], )) attk = CarliniWagnerL2(self.ch_model, sess=self.sess) params = { 'y': tf.one_hot([label], get_num_classes(self.dataset)), 'clip_min': 0.0, 'clip_max': 1.0, 'max_iterations': 1000 } adv_x = attk.generate(x_op, **params) adv_preds_op = self.tf_model(adv_x) (adv_preds, adv_xsamp) = self.sess.run((adv_preds_op, adv_x), feed_dict={x_op: xs}) adv_pred = np.argmax(adv_preds[0]) if adv_pred == label: # fail to find out adv example, return the radius to be the maximum one return la.norm(np.ones_like(adv_xsamp.reshape(-1)) * 0.5, 2) else: dist = la.norm(adv_xsamp.reshape(-1) - xs.numpy().reshape(-1), 2) return dist
args.beta = (1 - args.flip_alpha) / args.K ratio = args.beta / args.flip_alpha args.calibrated_alpha = args.flip_alpha - args.beta # args.calibrated_alpha = (1 - ratio) * args.alpha print('N / bz', args.N / args.batch) if __name__ == "__main__": # load the base classifier checkpoint = torch.load(args.base_classifier) print(checkpoint['epoch']) base_classifier = get_architecture(checkpoint["arch"], args.dataset) base_classifier.load_state_dict(checkpoint['state_dict']) # create the smooothed classifier g smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.calibrated_alpha, args.K) # prepare output file f = open(args.outfile, 'w') print("idx\tlabel\tpredict\tp_hat\trad.\tcorrect\ttime", file=f, flush=True) # iterate through the dataset dataset = get_dataset(args.dataset, args.split) for i in range(len(dataset)): # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i == args.max: break
def main(): iter_time = 2000 clients = [] test_accuracy_rate = [] average_loss = [] D_in = datasets.get_num_features("mnist") D_out = datasets.get_num_classes("mnist") batch_size = 10 train_cut = 0.8 for i in range(6): model = returnModel(D_in, D_out) clients.append( Client("mnist", "mnist" + str(i), batch_size, model, train_cut)) for i in range(4): model = returnModel(D_in, D_out) clients.append( Client("mnist", "mnist_bad_full", batch_size, model, train_cut)) model = returnModel(D_in, D_out) test_client = Client("mnist", "mnist_test", batch_size, model, 0) rejections = np.zeros(10) for iter in range(iter_time): modelWeights = clients[0].getModelWeights() # Calculate and aggregaate gradients for i in range(10): grad = clients[i].getGrad() # roni = test_client.roni(modelWeights, grad) # print "Client " + str(i) + " RONI is " + str(roni) # if roni > 0.02: # rejections[i] += 1 clients[0].updateGrad(grad) # Share updated model clients[0].step() modelWeights = clients[0].getModelWeights() for i in range(10): clients[i].updateModel(modelWeights) # Print average loss across clients if iter % 100 == 0: loss = 0.0 for i in range(10): loss += clients[i].getLoss() test_client.updateModel(modelWeights) test_err = test_client.getTestErr() attack_rate = test_client.get17AttackRate() print("Average loss is " + str(loss / len(clients))) print("Test error: " + str(test_err)) print("Attack rate on 1s: " + str(attack_rate) + "\n") average_loss.append(loss / len(clients)) test_accuracy_rate.append(attack_rate) # plot average loss and accuracy rate of the updating model x = range(1, int(math.floor(iter_time / 100)) + 1) fig, ax1 = plt.subplots() ax1.plot(x, average_loss, color='orangered', label='mnist_average_loss') plt.legend(loc=2) ax2 = ax1.twinx() ax2.plot(x, test_accuracy_rate, color='blue', label='mnist_test_accuracy_rate') plt.legend(loc=1) ax1.set_xlabel("iteration time / 100") ax1.set_ylabel("average_loss") ax2.set_ylabel("accuracy_rate") plt.title("mnist_graph") plt.legend() mp.show() test_client.updateModel(modelWeights) test_err = test_client.getTestErr() print("Test error: " + str(test_err)) accuracy_rate = 1 - test_err print("Accuracy rate: " + str(accuracy_rate))
setproctitle.setproctitle( f'rotation_certify_{args.dataset}from{args.start}to{args.max}') # prepare output file if not os.path.exists(os.path.dirname(args.outfile)): os.makedirs(os.path.dirname(args.outfile)) f = open(args.outfile, 'w') print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True) # init tensorboard writer writer = SummaryWriter(os.path.dirname(args.outfile)) # create the smooothed classifier g smoothed_classifier = SemanticSmooth(base_classifier, get_num_classes(args.dataset), transformer) tot, tot_clean, tot_good, tot_cert = 0, 0, 0, 0 for i in range(len(dataset)): if i < args.start: continue # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i >= args.max >= 0: break
parser.add_argument("--skip", type=int, default=1, help="how many examples to skip") parser.add_argument("--max", type=int, default=-1, help="stop after this many examples") parser.add_argument("--split", choices=["train", "test"], default="test", help="train or test set") parser.add_argument("--N0", type=int, default=100) parser.add_argument("--N", type=int, default=100000, help="number of samples to use") parser.add_argument("--alpha", type=float, default=0.001, help="failure probability") args = parser.parse_args() if __name__ == "__main__": # load the base classifier checkpoint = torch.load(args.base_classifier) base_classifier = get_architecture(checkpoint["arch"], args.dataset) base_classifier.load_state_dict(checkpoint['state_dict']) # create the smooothed classifier g smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.sigma) # prepare output file if not os.path.exists(os.path.dirname(args.outfile)): os.makedirs(os.path.dirname(args.outfile)) f = open(args.outfile, 'w') print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True) # iterate through the dataset dataset = get_dataset(args.dataset, args.split) for i in range(len(dataset)): # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i == args.max:
def __init__(self, dataset, model): super(CrownAdaptorBase, self).__init__(dataset, model) self.num_class = get_num_classes(dataset) self.new_model = BoundSequential.convert(self.model, {'same-slope': False})
args = get_arguments(sys.argv[1:]) # reading the config cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) # initialising the dirs check_dir(args.mask_output_dir, "vis") check_dir(args.mask_output_dir, "cs") if D_SAVE_RAW: check_dir(args.mask_output_dir, "raw") #check_dir(args.mask_output_dir, "crf") num_classes = get_num_classes(args) # Loading the model model = get_model(cfg.MODEL, 0, num_classes=num_classes) assert os.path.isfile(args.resume), "Snapshot not found: {}".format(args.resume) state_dict = convert_dict(torch.load(args.resume)["model"]) print(model) model.load_state_dict(state_dict, strict=False) for p in model.parameters(): p.requires_grad = False # setting the evaluation mode model.eval() model = nn.DataParallel(model).cuda()
def main(args): if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # load the base classifier checkpoint = torch.load(args.base_classifier) base_classifier = get_architecture(checkpoint["arch"], args.dataset) print('arch:', checkpoint['arch']) if checkpoint["arch"] == 'resnet50' and args.dataset == "imagenet": try: base_classifier.load_state_dict(checkpoint['state_dict']) except Exception as e: print('direct load failed, try alternative') try: base_classifier = torchvision.models.resnet50( pretrained=False).cuda() base_classifier.load_state_dict(checkpoint['state_dict']) # fix # normalize_layer = get_normalize_layer('imagenet').cuda() # base_classifier = torch.nn.Sequential(normalize_layer, base_classifier) except Exception as e: print('alternative failed again, try alternative 2') base_classifier = torchvision.models.resnet50( pretrained=False).cuda() # base_classifier.load_state_dict(checkpoint['state_dict']) normalize_layer = get_normalize_layer('imagenet').cuda() base_classifier = torch.nn.Sequential(normalize_layer, base_classifier) base_classifier.load_state_dict(checkpoint['state_dict']) else: base_classifier.load_state_dict(checkpoint['state_dict']) # iterate through the dataset dataset = get_dataset(args.dataset, args.split) # generate transformer transformer = gen_inference_transformer(args, dataset[0][0]) smoothed_classifier = SemanticSmooth(base_classifier, get_num_classes(args.dataset), transformer) # generate image-level transform and params tinst1, tfunc1, tinst2, tfunc2, param1l, param1r, param2l, param2r, candidates = gen_transform_and_params( args, dataset[0][0]) # init random number generator # m1 = Uniform(param1l, param1r) # if param2l is not None: # m2 = Uniform(param2l, param2r) # m1 = Uniform(param1l, param1r) m1 = Beta(0.5, 0.5) if param2l is not None: m2 = Beta(0.5, 0.5) # m2 = Uniform(param2l, param2r) # init metrics tot = tot_benign = tot_robust = 0 # [main] attack section for i in range(len(dataset)): # only certify every args.skip examples if i % args.skip != 0: continue print('working on #', i) (x, y) = dataset[i] pred = predict(smoothed_classifier, base_classifier, args, x) if pred != y: pass else: tot_benign += 1 robust = True for j in range(0, args.tries): param_sample1 = (m1.sample() * (param1r - param1l) + param1l).item() if param2l is not None: param_sample2 = (m2.sample() * (param2r - param2l) + param2l).item() else: param_sample2 = None xp = process(x, tfunc1, tinst1, tfunc2, tinst2, param_sample1, param_sample2) pre_loss = getloss(smoothed_classifier, base_classifier, args, xp, y) if param_sample2 is None: print( f"{i} > {j}/{args.tries} begin para1={param_sample1:4.2f} loss={pre_loss}", flush=True) else: print( f"{i} > {j}/{args.tries} begin para1={param_sample1:4.2f} para2={param_sample2:4.2f} loss={pre_loss}", flush=True) # first work on param1 eps = (param1r - param1l) / args.stepdiv xp_l = process(x, tfunc1, tinst1, tfunc2, tinst2, min(max(param_sample1 - eps, param1l), param1r), param_sample2) xp_r = process(x, tfunc1, tinst1, tfunc2, tinst2, min(max(param_sample1 + eps, param1l), param1r), param_sample2) loss_l = getloss(smoothed_classifier, base_classifier, args, xp_l, y) loss_r = getloss(smoothed_classifier, base_classifier, args, xp_r, y) coef_1 = 1 if loss_r > loss_l else -1 now_loss = max(loss_l, loss_r) now_param1 = param_sample1 if now_loss > pre_loss: while True: incre = min(max(now_param1 + coef_1 * eps, param1l), param1r) new_xp = process(x, tfunc1, tinst1, tfunc2, tinst2, incre, param_sample2) new_loss = getloss(smoothed_classifier, base_classifier, args, new_xp, y) # print(f"{i} > {j}/{args.tries} iter para1={now_param1 + coef_1 * eps} loss={new_loss}", flush=True) if new_loss < now_loss or ( not param1l < incre < param1r): break now_param1 = incre now_loss = new_loss tmp_l = now_param1 - coef_1 * eps tmp_r = now_param1 + coef_1 * eps tmp_l = min(max(tmp_l, param1l), param1r) tmp_r = min(max(tmp_r, param1l), param1r) # tri-section search while tmp_r - tmp_l > eps / args.stepdiv: tmp_m1 = (2.0 * tmp_l + tmp_r) / 3.0 tmp_m2 = (tmp_l + 2.0 * tmp_r) / 3.0 xp_m1 = process(x, tfunc1, tinst1, tfunc2, tinst2, tmp_m1, param_sample2) xp_m2 = process(x, tfunc1, tinst1, tfunc2, tinst2, tmp_m2, param_sample2) loss_m1 = getloss(smoothed_classifier, base_classifier, args, xp_m1, y) loss_m2 = getloss(smoothed_classifier, base_classifier, args, xp_m2, y) # print(f"{i} > {j}/{args.tries} search para1={tmp_m1} loss={loss_m1}", flush=True) # print(f"{i} > {j}/{args.tries} search para1={tmp_m2} loss={loss_m2}", flush=True) if loss_m1 > loss_m2: tmp_r = tmp_m2 else: tmp_l = tmp_m1 targ_param1 = (tmp_l + tmp_r) / 2.0 # now work on param2 if tfunc2 is not None: eps = (param2r - param2l) / args.stepdiv xp = process(x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, param_sample2) pre_loss2 = getloss(smoothed_classifier, base_classifier, args, xp, y) xp_l = process( x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, min(max(param_sample2 - eps, param2l), param2r)) xp_r = process( x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, min(max(param_sample2 + eps, param2l), param2r)) loss_l = getloss(smoothed_classifier, base_classifier, args, xp_l, y) loss_r = getloss(smoothed_classifier, base_classifier, args, xp_r, y) coef_2 = 1 if loss_r > loss_l else -1 now_loss = max(loss_l, loss_r) now_param2 = param_sample2 if now_loss > pre_loss2: while True: incre = min( max(now_param2 + coef_2 * eps, param2l), param2r) new_xp = process(x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, incre) new_loss = getloss(smoothed_classifier, base_classifier, args, new_xp, y) if new_loss < now_loss or ( not param2l < incre < param2r): break now_param2 = incre now_loss = new_loss tmp_l = now_param2 - coef_2 * eps tmp_r = now_param2 + coef_2 * eps tmp_l = min(max(tmp_l, param2l), param2r) tmp_r = min(max(tmp_r, param2l), param2r) # tri-section search while tmp_r - tmp_l > eps / args.stepdiv: tmp_m1 = (2.0 * tmp_l + tmp_r) / 3.0 tmp_m2 = (tmp_l + 2.0 * tmp_r) / 3.0 xp_m1 = process(x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, tmp_m1) xp_m2 = process(x, tfunc1, tinst1, tfunc2, tinst2, targ_param1, tmp_m2) loss_m1 = getloss(smoothed_classifier, base_classifier, args, xp_m1, y) loss_m2 = getloss(smoothed_classifier, base_classifier, args, xp_m2, y) if loss_m1 > loss_m2: tmp_r = tmp_m2 else: tmp_l = tmp_m1 targ_param2 = (tmp_l + tmp_r) / 2.0 xp = tfunc1(tinst1, x, targ_param1) if param2l is not None: xp = tfunc2(tinst2, xp, targ_param2) xp = xp.type_as(x) fin_loss = getloss(smoothed_classifier, base_classifier, args, xp, y) if param_sample2 is None: print( f"{i} > {j}/{args.tries} end para1={targ_param1:4.2f} loss={fin_loss}", flush=True) else: print( f"{i} > {j}/{args.tries} end para1={targ_param1:4.2f} para2={targ_param2:4.2f} loss={fin_loss}", flush=True) if args.transtype in [ 'rotation-brightness-l2', 'scaling-brightness-l2' ]: # compute the gradient by soft label and empirical mean smoothed_classifier.base_classifier.eval() grad = torch.zeros( (args.N0, xp.shape[0], xp.shape[1], xp.shape[2])) n = 0 while n < args.N0: now_batch = min(args.batch, args.N0 - n) if args.nosmooth is True: batch_noised = xp.repeat((1, 1, 1, 1)) else: batch = xp.repeat((now_batch, 1, 1, 1)) batch_noised = smoothed_classifier.transformer.process( batch).cuda() batch_noised = Variable( batch_noised.data, requires_grad=True).contiguous() opt = torch.optim.Adam([batch_noised], lr=1e-3) opt.zero_grad() loss = torch.nn.CrossEntropyLoss()( smoothed_classifier.base_classifier(batch_noised), torch.tensor( [y], dtype=torch.long).expand(now_batch).cuda()) loss.backward() grad[n:n + now_batch, :, :, :] = batch_noised.grad.data n += now_batch grad = torch.mean(grad, dim=0) unit_grad = F.normalize(grad, p=2, dim=list(range(grad.dim()))) delta = unit_grad * args.l2_r # print(xp) xp = xp + delta # print(xp + delta) # print(delta) # print(torch.norm(delta.reshape(-1))) if args.nosmooth is True: base_classifier.eval() pred = base_classifier(xp.cuda().unsqueeze(0)).argmax(1)[0] else: pred = smoothed_classifier.predict(xp, args.N, args.p, args.batch) if (pred != y): robust = False break print(f"{i} > {j}/{args.tries}", flush=True) tot_robust += int(robust) tot += 1 print( f'#{i} clean acc={tot_benign / tot} robust acc={tot_robust / tot}') if args.outfile is None: param_str = '' if args.transtype != 'translation': param_str = f'{param1r}' if param2r is not None: param_str += f'_{param2r}' else: param_str = f'{args.displacement}' args.outfile = args.transtype + '/' + args.dataset + '/' + param_str + '/' + 'result.txt' out_full_path = os.path.join(args.outfolder, args.outfile) print('output result to ' + out_full_path) if not os.path.exists(os.path.dirname(out_full_path)): os.makedirs(os.path.dirname(out_full_path)) f = open(out_full_path, 'a') f.write( f'clean {tot_benign / tot},{tot_benign} robust={tot_robust / tot},{tot_robust} tot={tot}\n' ) f.close() print('done')
def main(): if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu if not os.path.exists(args.outdir): os.makedirs(args.outdir) # Copies files to the outdir to store complete script with each experiment copy_code(args.outdir) train_dataset = get_dataset(args.dataset, 'train', rot=args.rot) test_dataset = get_dataset(args.dataset, 'test', rot=args.rot) pin_memory = (args.dataset in ["imagenet", "restricted_imagenet"]) labelled_loader = DataLoader(train_dataset, shuffle=True, batch_size=args.batch, num_workers=args.workers, pin_memory=pin_memory) if args.use_unlabelled: pseudo_labelled_loader = DataLoader(TiTop50KDataset(), shuffle=True, batch_size=args.batch, num_workers=args.workers, pin_memory=pin_memory) train_loader = MultiDatasetsDataLoader( [labelled_loader, pseudo_labelled_loader]) else: train_loader = MultiDatasetsDataLoader([labelled_loader]) test_loader = DataLoader(test_dataset, shuffle=False, batch_size=args.batch, num_workers=args.workers, pin_memory=pin_memory) if args.pretrained_model != '': assert args.arch == 'cifar_resnet110', 'Unsupported architecture for pretraining' checkpoint = torch.load(args.pretrained_model) model = get_architecture(checkpoint["arch"], args.dataset) model.load_state_dict(checkpoint['state_dict']) model[1].fc = nn.Linear(64, get_num_classes('cifar10')).cuda() else: model = get_architecture(args.arch, args.dataset) if args.attack == 'PGD': print('Attacker is PGD') attacker = PGD_L2(steps=args.num_steps, device='cuda', max_norm=args.epsilon) elif args.attack == 'DDN': print('Attacker is DDN') attacker = DDN(steps=args.num_steps, device='cuda', max_norm=args.epsilon, init_norm=args.init_norm_DDN, gamma=args.gamma_DDN) else: raise Exception('Unknown attack') criterion = CrossEntropyLoss().cuda() optimizer = SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = StepLR(optimizer, step_size=args.lr_step_size, gamma=args.gamma) starting_epoch = 0 logfilename = os.path.join(args.outdir, 'log.txt') # Load latest checkpoint if exists (to handle philly failures) model_path = os.path.join(args.outdir, 'checkpoint.pth.tar') if args.resume: if os.path.isfile(model_path): print("=> loading checkpoint '{}'".format(model_path)) checkpoint = torch.load(model_path, map_location=lambda storage, loc: storage) starting_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( model_path, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(model_path)) if args.adv_training: init_logfile( logfilename, "epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc\ttestaccNor" ) else: init_logfile( logfilename, "epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc") else: if args.adv_training: init_logfile( logfilename, "epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc\ttestaccNor" ) else: init_logfile( logfilename, "epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc") for epoch in range(starting_epoch, args.epochs): scheduler.step(epoch) attacker.max_norm = np.min( [args.epsilon, (epoch + 1) * args.epsilon / args.warmup]) attacker.init_norm = np.min( [args.epsilon, (epoch + 1) * args.epsilon / args.warmup]) before = time.time() train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, args.noise_sd, attacker) test_loss, test_acc, test_acc_normal = test(test_loader, model, criterion, args.noise_sd, attacker) after = time.time() if args.adv_training: log( logfilename, "{}\t{:.2f}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format( epoch, after - before, scheduler.get_lr()[0], train_loss, test_loss, train_acc, test_acc, test_acc_normal)) else: log( logfilename, "{}\t{:.2f}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format( epoch, after - before, scheduler.get_lr()[0], train_loss, test_loss, train_acc, test_acc)) torch.save( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }, model_path)
def main(args): if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # load the base classifier checkpoint = torch.load(args.base_classifier) base_classifier = get_architecture(checkpoint["arch"], args.dataset) print('arch:', checkpoint['arch']) if checkpoint["arch"] == 'resnet50' and args.dataset == "imagenet": try: base_classifier.load_state_dict(checkpoint['state_dict']) except Exception as e: print('direct load failed, try alternative') try: base_classifier = torchvision.models.resnet50( pretrained=False).cuda() base_classifier.load_state_dict(checkpoint['state_dict']) # fix # normalize_layer = get_normalize_layer('imagenet').cuda() # base_classifier = torch.nn.Sequential(normalize_layer, base_classifier) except Exception as e: print('alternative failed again, try alternative 2') base_classifier = torchvision.models.resnet50( pretrained=False).cuda() # base_classifier.load_state_dict(checkpoint['state_dict']) normalize_layer = get_normalize_layer('imagenet').cuda() base_classifier = torch.nn.Sequential(normalize_layer, base_classifier) base_classifier.load_state_dict(checkpoint['state_dict']) else: base_classifier.load_state_dict(checkpoint['state_dict']) # iterate through the dataset dataset = get_dataset(args.dataset, args.split) # generate transformer transformer = gen_inference_transformer(args, dataset[0][0]) smoothed_classifier = SemanticSmooth(base_classifier, get_num_classes(args.dataset), transformer) # generate image-level transform and params tinst1, tfunc1, tinst2, tfunc2, param1l, param1r, param2l, param2r, candidates = gen_transform_and_params( args, dataset[0][0]) # init random number generator m1 = Uniform(param1l, param1r) if param2l is not None: m2 = Uniform(param2l, param2r) # init metrics tot = tot_benign = tot_robust = 0 # [main] attack section for i in range(len(dataset)): # only certify every args.skip examples if i % args.skip != 0: continue print('working on #', i) (x, y) = dataset[i] # clean_x = x.cuda().unsqueeze(0) pred = smoothed_classifier.predict(x, args.N0, args.p, args.batch) if pred != y: pass else: tot_benign += 1 robust = True for j in range(0, args.tries): xp = None if args.transtype == 'translation': param_sample1 = candidates[int(m1.sample().item())] xp = tfunc1(tinst1, x, param_sample1[0].item(), param_sample1[1].item()) else: param_sample1 = m1.sample().item() if param2l is not None: param_sample2 = m2.sample().item() xp = tfunc1(tinst1, x, param_sample1) if param2l is not None: xp = tfunc2(tinst2, xp, param_sample2) # xp = xp.contiguous().cuda() # xp_old = xp # if args.l2 is not None and args.l2 > EPS: # xp = fgsm(model, xp, torch.tensor([y], dtype=torch.long).expand(now_batch).cuda(), args.l2) # print(torch.norm((xp_old - xp).reshape(xp.size()[0], -1), dim=1)) xp = xp.type_as(x) if args.transtype in [ 'rotation-brightness-l2', 'scaling-brightness-l2' ]: # compute the gradient by soft label and empirical mean smoothed_classifier.base_classifier.eval() grad = torch.zeros( (args.N0, xp.shape[0], xp.shape[1], xp.shape[2])) n = 0 while n < args.N0: now_batch = min(args.batch, args.N0 - n) batch = xp.repeat((now_batch, 1, 1, 1)) batch_noised = smoothed_classifier.transformer.process( batch).cuda() batch_noised = Variable( batch_noised.data, requires_grad=True).contiguous() opt = torch.optim.Adam([batch_noised], lr=1e-3) opt.zero_grad() loss = torch.nn.CrossEntropyLoss()( smoothed_classifier.base_classifier(batch_noised), torch.tensor( [y], dtype=torch.long).expand(now_batch).cuda()) loss.backward() grad[n:n + now_batch, :, :, :] = batch_noised.grad.data n += now_batch grad = torch.mean(grad, dim=0) unit_grad = F.normalize(grad, p=2, dim=list(range(grad.dim()))) delta = unit_grad * args.l2_r # print(xp) xp = xp + delta # print(xp + delta) # print(delta) # print(torch.norm(delta.reshape(-1))) pred = smoothed_classifier.predict(xp, args.N0, args.p, args.batch) if (pred != y): robust = False break print(f"> {j}/{args.tries}", end='\r', flush=True) tot_robust += int(robust) tot += 1 print( f'#{i} clean acc={tot_benign / tot} robust acc={tot_robust / tot}') if args.outfile is None: param_str = '' if args.transtype != 'translation': param_str = f'{param1r}' if param2r is not None: param_str += f'_{param2r}' else: param_str = f'{args.displacement}' args.outfile = args.transtype + '/' + args.dataset + '/' + param_str + '/' + 'result.txt' out_full_path = os.path.join(args.outfolder, args.outfile) print('output result to ' + out_full_path) if not os.path.exists(os.path.dirname(out_full_path)): os.makedirs(os.path.dirname(out_full_path)) f = open(out_full_path, 'w') f.write( f'clean {tot_benign / tot},{tot_benign} robust={tot_robust / tot},{tot_robust} tot={tot}\n' ) f.close() print('done')
parser.add_argument("--N", type=int, default=100000, help="number of samples to use") parser.add_argument("--alpha", type=float, default=0.001, help="failure probability") args = parser.parse_args() if __name__ == "__main__": # load the base classifier checkpoint = torch.load(args.base_classifier, map_location=torch.device('cpu')) base_classifier = get_architecture(checkpoint["arch"], args.dataset) base_classifier.load_state_dict(checkpoint['state_dict']) # create the smooothed classifier g smoothed_classifier = Smooth( base_classifier, get_num_classes(args.dataset), args.sigma) # prepare output file f = open(args.outfile, 'w') print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True) # iterate through the dataset dataset = get_dataset(args.dataset, args.split) for i in range(len(dataset)): # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i == args.max: break
def prologue(args): if not hasattr(args, 'id') or args.id is None: args.id = np.random.randint(10000) args.outdir = args.outdir + f"/{args.arch}/{args.id}/" if not os.path.exists(args.outdir): os.makedirs(args.outdir) # Copies files to the outdir to store complete script with each experiment copy_code(args.outdir) train_dataset = get_dataset(args.dataset, 'train') test_dataset = get_dataset(args.dataset, 'test') pin_memory = (args.dataset == "imagenet") train_loader = DataLoader(train_dataset, shuffle=True, batch_size=args.batch, num_workers=args.workers, pin_memory=pin_memory) test_loader = DataLoader(test_dataset, shuffle=False, batch_size=args.batch, num_workers=args.workers, pin_memory=pin_memory) if args.pretrained_model != '': assert args.arch == 'cifar_resnet110', 'Unsupported architecture for pretraining' checkpoint = torch.load(args.pretrained_model) model = get_architecture(checkpoint["arch"], args.dataset) model.load_state_dict(checkpoint['state_dict']) model[1].fc = nn.Linear(64, get_num_classes('cifar10')).to(device) else: model = get_architecture(args.arch, args.dataset) logfilename = os.path.join(args.outdir, 'log.txt') init_logfile(logfilename, "epoch\ttime\tlr\ttrain loss\ttrain acc\ttestloss\ttest acc") writer = SummaryWriter(args.outdir) criterion = CrossEntropyLoss().to(device) optimizer = SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = StepLR(optimizer, step_size=args.lr_step_size, gamma=args.gamma) starting_epoch = 0 # Load latest checkpoint if exists (to handle philly failures) model_path = os.path.join(args.outdir, 'checkpoint.pth.tar') if args.resume: if os.path.isfile(model_path): print("=> loading checkpoint '{}'".format(model_path)) checkpoint = torch.load(model_path, map_location=lambda storage, loc: storage) starting_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( model_path, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(model_path)) return train_loader, test_loader, criterion, model, optimizer, scheduler, \ starting_epoch, logfilename, model_path, device, writer
dataset = get_dataset(args.dataset, args.split) for i in range(len(dataset)): # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i == args.max: break (x, label) = dataset[i] #Smooth the classifier with this sigma if not args.fix_sig_smooth: args.sigma = sigma_test[i].item() print('sigma is: ', args.sigma) smoothed_classifier = Smooth(model, get_num_classes(args.dataset), args.sigma) #Now you can use the same exac before_time = time() # certify the prediction of g around x x = x.cuda() prediction, radius = smoothed_classifier.certify( x, args.N0, args.N, args.alpha, args.batch) after_time = time() correct = int(prediction == label) print(radius) time_elapsed = str( datetime.timedelta(seconds=(after_time - before_time))) print("{}\t{}\t{}\t{:.3}\t{}\t{:.3}\t{}".format( i, label, prediction, radius, correct, args.sigma, time_elapsed), file=f,
help="number of samples to use") parser.add_argument("--alpha", type=float, default=0.001, help="failure probability") args = parser.parse_args() if __name__ == "__main__": # load the base classifier checkpoint = torch.load(args.base_classifier) base_classifier = get_architecture(checkpoint["arch"], args.dataset) base_classifier.load_state_dict(checkpoint['state_dict']) # create the smoothed classifier g smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.sigma) # prepare output file f = open(args.outfile, 'w') # iterate through the dataset dataset = get_dataset(args.dataset, args.split) print("idx\tlabel\tpredict\tcorrect\tscore\ttime", flush=True) print("idx\tlabel\tpredict\tcorrect\tscore\ttime", file=f, flush=True) for i in range(len(dataset)): # only certify every args.skip examples, and stop after args.max examples if i % args.skip != 0: continue if i == args.max: break
def main(): iter_time = 1500 clients = [] average_loss = [] test_accuracy_rate = [] D_in = datasets.get_num_features("lfw") D_out = datasets.get_num_classes("lfw") batch_size = 4 train_cut = 0.8 print("Creating clients") for i in range(10): model = returnModel(D_in, D_out) clients.append( Client("lfw", "lfw_maleness_train" + str(i), batch_size, model, train_cut)) model = returnModel(D_in, D_out) test_client = Client("lfw", "lfw_maleness_test", batch_size, model, 0) print("Training for iterations") for iter in range(iter_time): # Calculate and aggregaate gradients for i in range(10): clients[0].updateGrad(clients[i].getGrad()) # Share updated model clients[0].step() modelWeights = clients[0].getModelWeights() for i in range(10): clients[i].updateModel(modelWeights) # Print average loss across clients if iter % 100 == 0: loss = 0.0 for i in range(10): loss += clients[i].getLoss() print("Average loss is " + str(loss / len(clients))) test_client.updateModel(modelWeights) test_err = test_client.getTestErr() print("Test error: " + str(test_err)) accuracy_rate = 1 - test_err print("Accuracy rate: " + str(accuracy_rate) + "\n") average_loss.append(loss / len(clients)) test_accuracy_rate.append(accuracy_rate) # plot average loss and accuracy rate of the updating model x = range(1, int(math.floor(iter_time / 100)) + 1) fig, ax1 = plt.subplots() ax1.plot(x, average_loss, color='orangered', label='lfw_gender_average_loss') plt.legend(loc=2) ax2 = ax1.twinx() ax2.plot(x, test_accuracy_rate, color='blue', label='lfw_gender_test_accuracy_rate') plt.legend(loc=1) ax1.set_xlabel("iteration time / 100") ax1.set_ylabel("average_loss") ax2.set_ylabel("accuracy_rate") plt.title("lfw_gender_graph") plt.legend() mp.show() test_client.updateModel(modelWeights) test_err = test_client.getTestErr() print("Test error: " + str(test_err)) accuracy_rate = 1 - test_err print("Accuracy rate: " + str(accuracy_rate) + "\n")