def __init__(self, dataset, model):
super(CNNCertBase, self).__init__(dataset, model)
self.num_classes = get_num_classes(dataset)
input_shape = get_input_shape(dataset)
new_input_shape = (input_shape[1], input_shape[2], input_shape[0])
self.k_model = sequential_torch2keras(model_transform(self.model, input_shape), dataset)
global graph
global sess
with sess.as_default():
with graph.as_default():
# Save the transformed Keras model to a temporary place so that the tool can read from file
# The tool can only init model from file...
sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True)
self.k_model.compile(loss=fn,
optimizer=sgd,
metrics=['accuracy'])
self.k_model.save('tmp/tmp.pt')
self.new_model = nl.CNNModel('tmp/tmp.pt', new_input_shape)
self.weights = self.new_model.weights
self.biases = self.new_model.biases
# print(self.new_model.summary())
try:
check_consistency(self.model, self.k_model, input_shape)
except Exception:
raise Exception("Somehow the transformed model behaves differently from the original model.")
self.LP = False
self.LPFULL = False
self.method = "ours"
self.dual = False
def verify(self, input, label, norm_type, radius):
"""
Here we overwrite the base class verify() method
"""
# only support Linfty norm
assert norm_type == 'inf'
# firstly check the clean prediction
input = self.input_preprocess(input)
xs = input.unsqueeze(0)
clean_preds = self.model(xs.cuda()).detach().cpu().numpy()
clean_pred = np.argmax(clean_preds[0])
if clean_pred != label:
return False
input = torch.Tensor(input)
m_radius = radius / self.coef
self.bound.calculate_bound(input, m_radius)
for i in range(get_num_classes(self.dataset)):
if i != label:
ok = self.bound.verify(label, i)
if not ok:
return False
return True
def main():
args = Args().get_args()
kwargs = vars(args)
checkpoint = torch.load(args.checkpoint)
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
base_classifier.load_state_dict(checkpoint['state_dict'])
attacker = SmoothAttack(base_classifier)
smoothed_classifier = Smooth(base_classifier,
get_num_classes(args.dataset), args.sigma)
dataset = get_dataset(args.dataset, 'test')
average_nat = []
average_adv = []
j_header('index', 'nat_y', 'adv_y', 'nat_rad', 'adv_rad', 'success')
figure = FigureSaver()
for i in range(0, len(dataset), args.skip):
(x, label) = dataset[i]
x = x.cuda()
first_x = x.data
nat_pred, nat_rad = smoothed_classifier.certify(
x, args.N0, args.N, args.alpha, args.batch)
if nat_pred is -1:
continue
if args.dataset == DATASETS[0]: # ImageNet
targets = [j for j in range(0, 1000, 100) if j is not label]
else:
targets = [j for j in range(10) if j is not label]
best_rad = -10.0
best_image = None
best_target = -1
for target in targets:
adv_x = attacker.perturb(x=first_x, y=target, **kwargs)
# If you want to do wasserstein attack, uncomment the following and change the attacker to wasserstein
# adv_x = attacker.perturb(x=first_x, y=target, eps=args.sigma, steps=args.steps, batch=args.batch)
adv_pred, adv_rad = smoothed_classifier.certify(
adv_x, args.N0, 2 * args.N0, args.alpha, args.batch)
adv_suc = (adv_pred != label) and (adv_pred != -1) and (nat_pred !=
-1)
adv_rad = adv_rad if adv_suc else -adv_rad
if adv_rad > best_rad:
best_rad = adv_rad
best_image = adv_x.data
best_target = target
figure.save(best_image, i, 'best={}'.format(best_target))
figure.save(first_x, i, 'natural')
best_pred, best_rad = smoothed_classifier.certify(
best_image, args.N0, args.N, args.alpha, args.batch)
j_print(i, label, best_target, nat_rad, best_rad)
average_adv.append(best_rad)
average_nat.append(nat_rad)
average_nat = np.array(average_nat)
average_adv = np.array(average_adv)
print('Average nat radii {}, Average adv radii {}'.format(
average_nat.mean(), average_adv.mean()))
def verify(self, input, label, norm_type, radius):
# only support Linfty norm
assert norm_type == 'inf'
if self.new_model is None:
# init at the first time
before = time()
print(f"Init model for {self.__class__.__name__}...")
in_shape = list(input.shape)
self.build_new_model(input)
self.prepare_solver(in_shape)
after = time()
print(
"Init done, time",
str(datetime.timedelta(seconds=(after - before))),
)
# firstly check the clean prediction
input = self.input_preprocess(input)
xs = input.unsqueeze(0)
clean_preds = self.model(xs.cuda()).detach().cpu().numpy()
clean_pred = np.argmax(clean_preds[0])
if clean_pred != label:
return False
m_radius = radius / self.coef
input = input.contiguous().view(-1)
self.bound.calculate_bound(input, m_radius)
for i in range(get_num_classes(self.dataset)):
if i != label:
ok = self.bound.verify(label, i)
if not ok:
return False
return True
def init(dataset, filename, epsilon, batch_size):
global myclient
D_in = datasets.get_num_features(dataset)
D_out = datasets.get_num_classes(dataset)
nParams = datasets.get_num_params(dataset)
model = SoftmaxModel(D_in, D_out)
train_cut = 0.8
myclient = client.Client(dataset, filename, batch_size, model, train_cut)
global samples
samples = []
global this_batch_size
this_batch_size = batch_size
def lnprob(x, alpha):
return -(alpha / 2) * np.linalg.norm(x)
if epsilon > 0:
if diffPriv13:
nwalkers = max(4 * nParams, 250)
sampler = emcee.EnsembleSampler(nwalkers,
nParams,
lnprob,
args=[epsilon])
p0 = [np.random.rand(nParams) for i in range(nwalkers)]
pos, _, state = sampler.run_mcmc(p0, 100)
sampler.reset()
sampler.run_mcmc(pos, 1000, rstate0=state)
print("Mean acceptance fraction:",
np.mean(sampler.acceptance_fraction))
samples = sampler.flatchain
else:
sigma = np.sqrt(2 * np.log(1.25)) / epsilon
noise = sigma * np.random.randn(batch_size, expected_iters,
nParams)
samples = np.sum(noise, axis=0)
else:
samples = np.zeros((expected_iters, nParams))
return nParams
def verify(self, input, label, norm_type, radius):
"""
Here we overwrite the base class verify() method
"""
# only support Linfty norm
assert norm_type == 'inf'
if self.new_model is None:
# init at the first time
before = time()
print(f"Init model for {self.__class__.__name__}...")
in_shape = list(input.shape)
self.build_new_model(input)
self.prepare_solver(in_shape)
after = time()
print(
"Init done, time",
str(datetime.timedelta(seconds=(after - before))),
)
# firstly check the clean prediction
input = self.input_preprocess(input)
xs = input.unsqueeze(0)
clean_preds = self.model(xs.cuda()).detach().cpu().numpy()
clean_pred = np.argmax(clean_preds[0])
if clean_pred != label:
return False
m_radius = radius / self.coef
input = input.contiguous().view(-1)
self.prebound.calculate_bound(input, m_radius)
self.bound.construct(self.prebound.l, self.prebound.u, input, m_radius)
for i in range(get_num_classes(self.dataset)):
if i != label:
self.bound.prepare_verify(label, i)
try:
# self.bound.prob.solve(verbose=True)
# model.setParam(GRB.Param.TimeLimit, timeout)
self.bound.prob.solve(solver=cp.GUROBI,
verbose=False,
BestObjStop=-1e-6,
TimeLimit=70,
Threads=20)
except:
return False
print(self.bound.prob.status,
self.bound.prob.value,
file=sys.stderr)
if self.bound.prob.status not in [
'optimal'
] or self.bound.prob.value < 0.:
return False
return True
def verify(self, input, label, norm_type, radius):
"""
Here we overwrite the base class verify() method
"""
in_shape = list(input.shape)
# only support Linfty norm
assert norm_type == 'inf'
if self.new_model is None:
# init at the first time
before = time()
print(f"Init model for {self.__class__.__name__}...")
self.build_new_model(input)
self.prepare_solver(in_shape)
after = time()
print(
"Init done, time",
str(datetime.timedelta(seconds=(after - before))),
)
# firstly check the clean prediction
input = self.input_preprocess(input)
xs = input.unsqueeze(0)
clean_preds = self.model(xs.cuda()).detach().cpu().numpy()
clean_pred = np.argmax(clean_preds[0])
if clean_pred != label:
return False
m_radius = radius / self.coef
input = input.contiguous().view(-1)
self.prebound.calculate_bound(input, m_radius)
bl = [
np.maximum(self.prebound.l[i], 0) if i > 0 else self.prebound.l[i]
for i in range(len(self.prebound.l))
]
bu = [
np.maximum(self.prebound.u[i], 0) if i > 0 else self.prebound.u[i]
for i in range(len(self.prebound.u))
]
pv = BaselinePointVerifierExt(self.new_model, in_shape, self.in_min,
self.in_max)
for i in range(get_num_classes(self.dataset)):
if i != label:
pv.create_cmat(input, label, i, m_radius, bl, bu)
pv.run()
if pv.prob.status not in ['unbounded', 'unbounded_inaccurate'
] and pv.prob.value > 0.:
return False
return True
def __init__(self, dataset, model):
super(CNNCertBase, self).__init__(dataset, model)
self.num_classes = get_num_classes(dataset)
self.activation = list()
for layer in self.model:
if isinstance(layer, nn.ReLU):
self.activation.append('ada')
elif isinstance(layer, nn.Sigmoid):
self.activation.append('sigmoid')
elif isinstance(layer, nn.Tanh):
self.activation.append('tanh')
# actually there is another activation called arctan,
# but there is no corresponding one in pytorch so we ignore it
self.activation = list(set(self.activation))
assert len(self.activation) == 1
self.activation = self.activation[0]
input_shape = get_input_shape(dataset)
new_input_shape = (input_shape[1], input_shape[2], input_shape[0])
self.k_model = sequential_torch2keras(self.model, dataset)
global graph
global sess
with sess.as_default():
with graph.as_default():
print(self.k_model.summary())
try:
assert check_consistency(self.model, self.k_model, input_shape) == True
except:
raise Exception("Somehow the transformed model behaves differently from the original model.")
self.new_model = Model(self.k_model, new_input_shape)
# Set correct linear_bounds function
self.linear_bounds = None
if self.activation == 'relu':
self.linear_bounds = relu_linear_bounds
elif self.activation == 'ada':
self.linear_bounds = ada_linear_bounds
elif self.activation == 'sigmoid':
self.linear_bounds = sigmoid_linear_bounds
elif self.activation == 'tanh':
self.linear_bounds = tanh_linear_bounds
elif self.activation == 'arctan':
self.linear_bounds = atan_linear_bounds
def __init__(self, args, **kwargs):
super(DecTrainer, self).__init__(args, **kwargs)
# dataloader
self.trainloader = get_dataloader(args, cfg, 'train')
# self.trainloader_val = get_dataloader(args, cfg, 'train_voc')
self.valloader = get_dataloader(args, cfg, 'val')
self.denorm = self.trainloader.dataset.denorm
self.use_triplet = args.use_triplet
self.loss_3d = args.loss_3d
self.normalize_feature = args.normalize_feature
self.nclass = get_num_classes(args)
self.classNames = get_class_names(args)
assert self.nclass == len(self.classNames) - 1
self.classIndex = {}
for i, cname in enumerate(self.classNames):
self.classIndex[cname] = i
# model
self.enc = get_model(cfg.NET, num_classes=self.nclass)
self.criterion_cls = get_criterion(cfg.NET.LOSS)
# optimizer using different LR
enc_params = self.enc.parameter_groups(cfg.NET.LR, cfg.NET.WEIGHT_DECAY)
self.optim_enc = self.get_optim(enc_params, cfg.NET)
# checkpoint management
self._define_checkpoint('enc', self.enc, self.optim_enc)
self._load_checkpoint(args.resume)
self.fixed_batch = None
self.fixed_batch_path = args.fixed_batch_path
if os.path.isfile(self.fixed_batch_path):
print("Loading fixed batch from {}".format(self.fixed_batch_path))
self.fixed_batch = torch.load(self.fixed_batch_path)
# using cuda
if cfg.NUM_GPUS != 0:
self.enc = nn.DataParallel(self.enc)
self.criterion_cls = nn.DataParallel(self.criterion_cls)
self.enc = self.enc.cuda()
self.criterion_cls = self.criterion_cls.cuda()
# CHANGE: visual
self.visual_times = 0
self.dataset = args.dataset.lower()
def verify(self, input, label, norm_type, radius):
assert norm_type == 'inf'
self.load_config()
input = self.input_preprocess(input)
m_radius = radius / self.coef
image = input.permute(1, 2, 0).contiguous().numpy().reshape(-1)
specLB = np.copy(image)
specUB = np.copy(image)
domain = config.domain
pred, nn, nlb, nub = self.eran.analyze_box(
specLB, specUB, init_domain(domain), config.timeout_lp,
config.timeout_milp, config.use_default_heuristic)
if pred == label:
specLB = np.clip(image - m_radius, self.in_min, self.in_max)
specUB = np.clip(image + m_radius, self.in_min, self.in_max)
perturbed_label, _, nlb, nub = self.eran.analyze_box(
specLB, specUB, domain, config.timeout_lp, config.timeout_milp,
config.use_default_heuristic)
# print("nlb ", nlb[len(nlb) - 1], " nub ", nub[len(nub) - 1])
if (perturbed_label == label):
return True
else:
if config.complete:
# print("complete verification here")
constraints = get_constraints_for_dominant_label(
label, get_num_classes(self.dataset))
verified_flag, adv_image = verify_network_with_milp(
nn, specLB, specUB, nlb, nub, constraints)
if (verified_flag == True):
return True
else:
return False
else:
return False
else:
raise Exception(
"Wrong prediction... This should not happen according to evaluate.py"
)
# print('Prediction is wrong')
return False
def train(loader: DataLoader, model: torch.nn.Module, optimizer: Optimizer,
epoch: int, noise_sd: float, device: torch.device, writer=None):
# switch to train mode
model.train()
lbd = args.lbd
if args.deferred and epoch <= args.lr_step_size:
lbd = 0
cl, rl = macer_train(sigma=noise_sd, lbd=lbd, gauss_num=args.num_noise_vec,
beta=args.beta, gamma=args.margin,
num_classes=get_num_classes(args.dataset),
model=model, trainloader=loader, optimizer=optimizer, device=device)
writer.add_scalar('loss/train', cl, epoch)
writer.add_scalar('loss/robust', rl, epoch)
return cl
def __init__(self, dataset, filename, train_cut=.80):
# initializes dataset
self.batch_size = 4
Dataset = datasets.get_dataset(dataset)
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
self.trainset = Dataset(filename,
"data/" + dataset,
is_train=True,
train_cut=train_cut,
transform=transform)
self.testset = Dataset(filename,
"data/" + dataset,
is_train=False,
train_cut=train_cut,
transform=transform)
self.trainloader = torch.utils.data.DataLoader(
self.trainset, batch_size=self.batch_size, shuffle=True)
self.testloader = torch.utils.data.DataLoader(self.testset,
batch_size=len(
self.testset),
shuffle=False)
D_in = datasets.get_num_features(dataset)
D_out = datasets.get_num_classes(dataset)
self.model = SoftmaxModel(D_in, D_out)
# self.model = MNISTCNNModel()
# self.model = LFWCNNModel()
# self.model = SVMModel(D_in, D_out)
# self.criterion = nn.MultiLabelMarginLoss()
### Tunables ###
self.criterion = nn.CrossEntropyLoss()
self.optimizer = optim.SGD(self.model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001)
self.aggregatedGradients = []
self.loss = 0.0
def __init__(self, args, **kwargs):
super(DecTrainer, self).__init__(args, **kwargs)
# dataloader
self.trainloader = get_dataloader(args, cfg, 'train')
self.trainloader_val = get_dataloader(args, cfg, 'train_voc')
self.valloader = get_dataloader(args, cfg, 'val')
self.denorm = self.trainloader.dataset.denorm
self.nclass = get_num_classes(args)
self.classNames = get_class_names(args)[:-1]
assert self.nclass == len(self.classNames)
self.classIndex = {}
for i, cname in enumerate(self.classNames):
self.classIndex[cname] = i
# model
self.enc = get_model(cfg.GENERATOR, num_classes=self.nclass)
self.criterion_cls = get_criterion(cfg.GENERATOR.LOSS)
print(self.enc)
# optimizer using different LR
enc_params = self.enc.parameter_groups(cfg.GENERATOR.LR, cfg.GENERATOR.WEIGHT_DECAY)
self.optim_enc = self.get_optim(enc_params, cfg.GENERATOR)
# checkpoint management
self._define_checkpoint('enc', self.enc, self.optim_enc)
self._load_checkpoint(args.resume)
self.fixed_batch = None
self.fixed_batch_path = args.fixed_batch_path
if os.path.isfile(self.fixed_batch_path):
print("Loading fixed batch from {}".format(self.fixed_batch_path))
self.fixed_batch = torch.load(self.fixed_batch_path)
# using cuda
self.enc = nn.DataParallel(self.enc).cuda()
self.criterion_cls = nn.DataParallel(self.criterion_cls).cuda()
def main(batchsize, epsilon, dataset, data_filename, dataclass):
train_cut = 0.8
iter_time = 1
clients = []
average_loss = []
test_accuracy_rate = []
D_in = datasets.get_num_features(dataset)
D_out = datasets.get_num_classes(dataset)
global batch_size
batch_size = batchsize
global nParams
nParams = datasets.get_num_params(dataset)
print("Creating clients")
# for i in range(10):
# model = returnModel(D_in, D_out)
# clients.append(Client("lfw", "lfw_maleness_train" + str(i), batch_size, model, train_cut))
model = returnModel(D_in, D_out)
# clients.append(Client("lfw", "lfw_maleness_person61_over20", batch_size, model, train_cut))
clients.append(Client(dataset, data_filename, batch_size, model,
train_cut))
print("Training for iterations")
for iter in range(iter_time):
# Calculate and aggregate gradients
for i in range(1):
grad = clients[i].getGrad()
if epsilon == 0:
showImage(grad, dataset, dataclass, batch_size, 0)
else:
varyEpsilonShowImage(grad, batch_size, epsilon, dataset,
data_class)
def calc_radius(self, input, label, norm_type, upper=0.5, eps=1e-2):
# only support L2 norm
assert norm_type == '2'
xs = input.unsqueeze(0)
clean_preds = self.model(xs.cuda()).detach().cpu().numpy()
clean_pred = np.argmax(clean_preds[0])
if clean_pred != label:
return 0.
x_op = tf.placeholder(tf.float32,
shape=(
None,
input.shape[0],
input.shape[1],
input.shape[2],
))
attk = CarliniWagnerL2(self.ch_model, sess=self.sess)
params = {
'y': tf.one_hot([label], get_num_classes(self.dataset)),
'clip_min': 0.0,
'clip_max': 1.0,
'max_iterations': 1000
}
adv_x = attk.generate(x_op, **params)
adv_preds_op = self.tf_model(adv_x)
(adv_preds, adv_xsamp) = self.sess.run((adv_preds_op, adv_x),
feed_dict={x_op: xs})
adv_pred = np.argmax(adv_preds[0])
if adv_pred == label:
# fail to find out adv example, return the radius to be the maximum one
return la.norm(np.ones_like(adv_xsamp.reshape(-1)) * 0.5, 2)
else:
dist = la.norm(adv_xsamp.reshape(-1) - xs.numpy().reshape(-1), 2)
return dist
args.beta = (1 - args.flip_alpha) / args.K
ratio = args.beta / args.flip_alpha
args.calibrated_alpha = args.flip_alpha - args.beta
# args.calibrated_alpha = (1 - ratio) * args.alpha
print('N / bz', args.N / args.batch)
if __name__ == "__main__":
# load the base classifier
checkpoint = torch.load(args.base_classifier)
print(checkpoint['epoch'])
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
base_classifier.load_state_dict(checkpoint['state_dict'])
# create the smooothed classifier g
smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.calibrated_alpha, args.K)
# prepare output file
f = open(args.outfile, 'w')
print("idx\tlabel\tpredict\tp_hat\trad.\tcorrect\ttime", file=f, flush=True)
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
for i in range(len(dataset)):
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i == args.max:
break
def main():
iter_time = 2000
clients = []
test_accuracy_rate = []
average_loss = []
D_in = datasets.get_num_features("mnist")
D_out = datasets.get_num_classes("mnist")
batch_size = 10
train_cut = 0.8
for i in range(6):
model = returnModel(D_in, D_out)
clients.append(
Client("mnist", "mnist" + str(i), batch_size, model, train_cut))
for i in range(4):
model = returnModel(D_in, D_out)
clients.append(
Client("mnist", "mnist_bad_full", batch_size, model, train_cut))
model = returnModel(D_in, D_out)
test_client = Client("mnist", "mnist_test", batch_size, model, 0)
rejections = np.zeros(10)
for iter in range(iter_time):
modelWeights = clients[0].getModelWeights()
# Calculate and aggregaate gradients
for i in range(10):
grad = clients[i].getGrad()
# roni = test_client.roni(modelWeights, grad)
# print "Client " + str(i) + " RONI is " + str(roni)
# if roni > 0.02:
# rejections[i] += 1
clients[0].updateGrad(grad)
# Share updated model
clients[0].step()
modelWeights = clients[0].getModelWeights()
for i in range(10):
clients[i].updateModel(modelWeights)
# Print average loss across clients
if iter % 100 == 0:
loss = 0.0
for i in range(10):
loss += clients[i].getLoss()
test_client.updateModel(modelWeights)
test_err = test_client.getTestErr()
attack_rate = test_client.get17AttackRate()
print("Average loss is " + str(loss / len(clients)))
print("Test error: " + str(test_err))
print("Attack rate on 1s: " + str(attack_rate) + "\n")
average_loss.append(loss / len(clients))
test_accuracy_rate.append(attack_rate)
# plot average loss and accuracy rate of the updating model
x = range(1, int(math.floor(iter_time / 100)) + 1)
fig, ax1 = plt.subplots()
ax1.plot(x, average_loss, color='orangered', label='mnist_average_loss')
plt.legend(loc=2)
ax2 = ax1.twinx()
ax2.plot(x,
test_accuracy_rate,
color='blue',
label='mnist_test_accuracy_rate')
plt.legend(loc=1)
ax1.set_xlabel("iteration time / 100")
ax1.set_ylabel("average_loss")
ax2.set_ylabel("accuracy_rate")
plt.title("mnist_graph")
plt.legend()
mp.show()
test_client.updateModel(modelWeights)
test_err = test_client.getTestErr()
print("Test error: " + str(test_err))
accuracy_rate = 1 - test_err
print("Accuracy rate: " + str(accuracy_rate))
setproctitle.setproctitle(
f'rotation_certify_{args.dataset}from{args.start}to{args.max}')
# prepare output file
if not os.path.exists(os.path.dirname(args.outfile)):
os.makedirs(os.path.dirname(args.outfile))
f = open(args.outfile, 'w')
print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True)
# init tensorboard writer
writer = SummaryWriter(os.path.dirname(args.outfile))
# create the smooothed classifier g
smoothed_classifier = SemanticSmooth(base_classifier,
get_num_classes(args.dataset),
transformer)
tot, tot_clean, tot_good, tot_cert = 0, 0, 0, 0
for i in range(len(dataset)):
if i < args.start:
continue
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i >= args.max >= 0:
break
parser.add_argument("--skip", type=int, default=1, help="how many examples to skip")
parser.add_argument("--max", type=int, default=-1, help="stop after this many examples")
parser.add_argument("--split", choices=["train", "test"], default="test", help="train or test set")
parser.add_argument("--N0", type=int, default=100)
parser.add_argument("--N", type=int, default=100000, help="number of samples to use")
parser.add_argument("--alpha", type=float, default=0.001, help="failure probability")
args = parser.parse_args()
if __name__ == "__main__":
# load the base classifier
checkpoint = torch.load(args.base_classifier)
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
base_classifier.load_state_dict(checkpoint['state_dict'])
# create the smooothed classifier g
smoothed_classifier = Smooth(base_classifier, get_num_classes(args.dataset), args.sigma)
# prepare output file
if not os.path.exists(os.path.dirname(args.outfile)):
os.makedirs(os.path.dirname(args.outfile))
f = open(args.outfile, 'w')
print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True)
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
for i in range(len(dataset)):
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i == args.max:
def __init__(self, dataset, model):
super(CrownAdaptorBase, self).__init__(dataset, model)
self.num_class = get_num_classes(dataset)
self.new_model = BoundSequential.convert(self.model, {'same-slope': False})
args = get_arguments(sys.argv[1:])
# reading the config
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
# initialising the dirs
check_dir(args.mask_output_dir, "vis")
check_dir(args.mask_output_dir, "cs")
if D_SAVE_RAW:
check_dir(args.mask_output_dir, "raw")
#check_dir(args.mask_output_dir, "crf")
num_classes = get_num_classes(args)
# Loading the model
model = get_model(cfg.MODEL, 0, num_classes=num_classes)
assert os.path.isfile(args.resume), "Snapshot not found: {}".format(args.resume)
state_dict = convert_dict(torch.load(args.resume)["model"])
print(model)
model.load_state_dict(state_dict, strict=False)
for p in model.parameters():
p.requires_grad = False
# setting the evaluation mode
model.eval()
model = nn.DataParallel(model).cuda()
def main(args):
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# load the base classifier
checkpoint = torch.load(args.base_classifier)
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
print('arch:', checkpoint['arch'])
if checkpoint["arch"] == 'resnet50' and args.dataset == "imagenet":
try:
base_classifier.load_state_dict(checkpoint['state_dict'])
except Exception as e:
print('direct load failed, try alternative')
try:
base_classifier = torchvision.models.resnet50(
pretrained=False).cuda()
base_classifier.load_state_dict(checkpoint['state_dict'])
# fix
# normalize_layer = get_normalize_layer('imagenet').cuda()
# base_classifier = torch.nn.Sequential(normalize_layer, base_classifier)
except Exception as e:
print('alternative failed again, try alternative 2')
base_classifier = torchvision.models.resnet50(
pretrained=False).cuda()
# base_classifier.load_state_dict(checkpoint['state_dict'])
normalize_layer = get_normalize_layer('imagenet').cuda()
base_classifier = torch.nn.Sequential(normalize_layer,
base_classifier)
base_classifier.load_state_dict(checkpoint['state_dict'])
else:
base_classifier.load_state_dict(checkpoint['state_dict'])
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
# generate transformer
transformer = gen_inference_transformer(args, dataset[0][0])
smoothed_classifier = SemanticSmooth(base_classifier,
get_num_classes(args.dataset),
transformer)
# generate image-level transform and params
tinst1, tfunc1, tinst2, tfunc2, param1l, param1r, param2l, param2r, candidates = gen_transform_and_params(
args, dataset[0][0])
# init random number generator
# m1 = Uniform(param1l, param1r)
# if param2l is not None:
# m2 = Uniform(param2l, param2r)
# m1 = Uniform(param1l, param1r)
m1 = Beta(0.5, 0.5)
if param2l is not None:
m2 = Beta(0.5, 0.5)
# m2 = Uniform(param2l, param2r)
# init metrics
tot = tot_benign = tot_robust = 0
# [main] attack section
for i in range(len(dataset)):
# only certify every args.skip examples
if i % args.skip != 0:
continue
print('working on #', i)
(x, y) = dataset[i]
pred = predict(smoothed_classifier, base_classifier, args, x)
if pred != y:
pass
else:
tot_benign += 1
robust = True
for j in range(0, args.tries):
param_sample1 = (m1.sample() * (param1r - param1l) +
param1l).item()
if param2l is not None:
param_sample2 = (m2.sample() * (param2r - param2l) +
param2l).item()
else:
param_sample2 = None
xp = process(x, tfunc1, tinst1, tfunc2, tinst2, param_sample1,
param_sample2)
pre_loss = getloss(smoothed_classifier, base_classifier, args,
xp, y)
if param_sample2 is None:
print(
f"{i} > {j}/{args.tries} begin para1={param_sample1:4.2f} loss={pre_loss}",
flush=True)
else:
print(
f"{i} > {j}/{args.tries} begin para1={param_sample1:4.2f} para2={param_sample2:4.2f} loss={pre_loss}",
flush=True)
# first work on param1
eps = (param1r - param1l) / args.stepdiv
xp_l = process(x, tfunc1, tinst1, tfunc2, tinst2,
min(max(param_sample1 - eps, param1l), param1r),
param_sample2)
xp_r = process(x, tfunc1, tinst1, tfunc2, tinst2,
min(max(param_sample1 + eps, param1l), param1r),
param_sample2)
loss_l = getloss(smoothed_classifier, base_classifier, args,
xp_l, y)
loss_r = getloss(smoothed_classifier, base_classifier, args,
xp_r, y)
coef_1 = 1 if loss_r > loss_l else -1
now_loss = max(loss_l, loss_r)
now_param1 = param_sample1
if now_loss > pre_loss:
while True:
incre = min(max(now_param1 + coef_1 * eps, param1l),
param1r)
new_xp = process(x, tfunc1, tinst1, tfunc2, tinst2,
incre, param_sample2)
new_loss = getloss(smoothed_classifier,
base_classifier, args, new_xp, y)
# print(f"{i} > {j}/{args.tries} iter para1={now_param1 + coef_1 * eps} loss={new_loss}", flush=True)
if new_loss < now_loss or (
not param1l < incre < param1r):
break
now_param1 = incre
now_loss = new_loss
tmp_l = now_param1 - coef_1 * eps
tmp_r = now_param1 + coef_1 * eps
tmp_l = min(max(tmp_l, param1l), param1r)
tmp_r = min(max(tmp_r, param1l), param1r)
# tri-section search
while tmp_r - tmp_l > eps / args.stepdiv:
tmp_m1 = (2.0 * tmp_l + tmp_r) / 3.0
tmp_m2 = (tmp_l + 2.0 * tmp_r) / 3.0
xp_m1 = process(x, tfunc1, tinst1, tfunc2, tinst2, tmp_m1,
param_sample2)
xp_m2 = process(x, tfunc1, tinst1, tfunc2, tinst2, tmp_m2,
param_sample2)
loss_m1 = getloss(smoothed_classifier, base_classifier,
args, xp_m1, y)
loss_m2 = getloss(smoothed_classifier, base_classifier,
args, xp_m2, y)
# print(f"{i} > {j}/{args.tries} search para1={tmp_m1} loss={loss_m1}", flush=True)
# print(f"{i} > {j}/{args.tries} search para1={tmp_m2} loss={loss_m2}", flush=True)
if loss_m1 > loss_m2:
tmp_r = tmp_m2
else:
tmp_l = tmp_m1
targ_param1 = (tmp_l + tmp_r) / 2.0
# now work on param2
if tfunc2 is not None:
eps = (param2r - param2l) / args.stepdiv
xp = process(x, tfunc1, tinst1, tfunc2, tinst2,
targ_param1, param_sample2)
pre_loss2 = getloss(smoothed_classifier, base_classifier,
args, xp, y)
xp_l = process(
x, tfunc1, tinst1, tfunc2, tinst2, targ_param1,
min(max(param_sample2 - eps, param2l), param2r))
xp_r = process(
x, tfunc1, tinst1, tfunc2, tinst2, targ_param1,
min(max(param_sample2 + eps, param2l), param2r))
loss_l = getloss(smoothed_classifier, base_classifier,
args, xp_l, y)
loss_r = getloss(smoothed_classifier, base_classifier,
args, xp_r, y)
coef_2 = 1 if loss_r > loss_l else -1
now_loss = max(loss_l, loss_r)
now_param2 = param_sample2
if now_loss > pre_loss2:
while True:
incre = min(
max(now_param2 + coef_2 * eps, param2l),
param2r)
new_xp = process(x, tfunc1, tinst1, tfunc2, tinst2,
targ_param1, incre)
new_loss = getloss(smoothed_classifier,
base_classifier, args, new_xp,
y)
if new_loss < now_loss or (
not param2l < incre < param2r):
break
now_param2 = incre
now_loss = new_loss
tmp_l = now_param2 - coef_2 * eps
tmp_r = now_param2 + coef_2 * eps
tmp_l = min(max(tmp_l, param2l), param2r)
tmp_r = min(max(tmp_r, param2l), param2r)
# tri-section search
while tmp_r - tmp_l > eps / args.stepdiv:
tmp_m1 = (2.0 * tmp_l + tmp_r) / 3.0
tmp_m2 = (tmp_l + 2.0 * tmp_r) / 3.0
xp_m1 = process(x, tfunc1, tinst1, tfunc2, tinst2,
targ_param1, tmp_m1)
xp_m2 = process(x, tfunc1, tinst1, tfunc2, tinst2,
targ_param1, tmp_m2)
loss_m1 = getloss(smoothed_classifier, base_classifier,
args, xp_m1, y)
loss_m2 = getloss(smoothed_classifier, base_classifier,
args, xp_m2, y)
if loss_m1 > loss_m2:
tmp_r = tmp_m2
else:
tmp_l = tmp_m1
targ_param2 = (tmp_l + tmp_r) / 2.0
xp = tfunc1(tinst1, x, targ_param1)
if param2l is not None:
xp = tfunc2(tinst2, xp, targ_param2)
xp = xp.type_as(x)
fin_loss = getloss(smoothed_classifier, base_classifier, args,
xp, y)
if param_sample2 is None:
print(
f"{i} > {j}/{args.tries} end para1={targ_param1:4.2f} loss={fin_loss}",
flush=True)
else:
print(
f"{i} > {j}/{args.tries} end para1={targ_param1:4.2f} para2={targ_param2:4.2f} loss={fin_loss}",
flush=True)
if args.transtype in [
'rotation-brightness-l2', 'scaling-brightness-l2'
]:
# compute the gradient by soft label and empirical mean
smoothed_classifier.base_classifier.eval()
grad = torch.zeros(
(args.N0, xp.shape[0], xp.shape[1], xp.shape[2]))
n = 0
while n < args.N0:
now_batch = min(args.batch, args.N0 - n)
if args.nosmooth is True:
batch_noised = xp.repeat((1, 1, 1, 1))
else:
batch = xp.repeat((now_batch, 1, 1, 1))
batch_noised = smoothed_classifier.transformer.process(
batch).cuda()
batch_noised = Variable(
batch_noised.data,
requires_grad=True).contiguous()
opt = torch.optim.Adam([batch_noised], lr=1e-3)
opt.zero_grad()
loss = torch.nn.CrossEntropyLoss()(
smoothed_classifier.base_classifier(batch_noised),
torch.tensor(
[y],
dtype=torch.long).expand(now_batch).cuda())
loss.backward()
grad[n:n + now_batch, :, :, :] = batch_noised.grad.data
n += now_batch
grad = torch.mean(grad, dim=0)
unit_grad = F.normalize(grad,
p=2,
dim=list(range(grad.dim())))
delta = unit_grad * args.l2_r
# print(xp)
xp = xp + delta
# print(xp + delta)
# print(delta)
# print(torch.norm(delta.reshape(-1)))
if args.nosmooth is True:
base_classifier.eval()
pred = base_classifier(xp.cuda().unsqueeze(0)).argmax(1)[0]
else:
pred = smoothed_classifier.predict(xp, args.N, args.p,
args.batch)
if (pred != y):
robust = False
break
print(f"{i} > {j}/{args.tries}", flush=True)
tot_robust += int(robust)
tot += 1
print(
f'#{i} clean acc={tot_benign / tot} robust acc={tot_robust / tot}')
if args.outfile is None:
param_str = ''
if args.transtype != 'translation':
param_str = f'{param1r}'
if param2r is not None:
param_str += f'_{param2r}'
else:
param_str = f'{args.displacement}'
args.outfile = args.transtype + '/' + args.dataset + '/' + param_str + '/' + 'result.txt'
out_full_path = os.path.join(args.outfolder, args.outfile)
print('output result to ' + out_full_path)
if not os.path.exists(os.path.dirname(out_full_path)):
os.makedirs(os.path.dirname(out_full_path))
f = open(out_full_path, 'a')
f.write(
f'clean {tot_benign / tot},{tot_benign} robust={tot_robust / tot},{tot_robust} tot={tot}\n'
)
f.close()
print('done')
def main():
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
# Copies files to the outdir to store complete script with each experiment
copy_code(args.outdir)
train_dataset = get_dataset(args.dataset, 'train', rot=args.rot)
test_dataset = get_dataset(args.dataset, 'test', rot=args.rot)
pin_memory = (args.dataset in ["imagenet", "restricted_imagenet"])
labelled_loader = DataLoader(train_dataset,
shuffle=True,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
if args.use_unlabelled:
pseudo_labelled_loader = DataLoader(TiTop50KDataset(),
shuffle=True,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
train_loader = MultiDatasetsDataLoader(
[labelled_loader, pseudo_labelled_loader])
else:
train_loader = MultiDatasetsDataLoader([labelled_loader])
test_loader = DataLoader(test_dataset,
shuffle=False,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
if args.pretrained_model != '':
assert args.arch == 'cifar_resnet110', 'Unsupported architecture for pretraining'
checkpoint = torch.load(args.pretrained_model)
model = get_architecture(checkpoint["arch"], args.dataset)
model.load_state_dict(checkpoint['state_dict'])
model[1].fc = nn.Linear(64, get_num_classes('cifar10')).cuda()
else:
model = get_architecture(args.arch, args.dataset)
if args.attack == 'PGD':
print('Attacker is PGD')
attacker = PGD_L2(steps=args.num_steps,
device='cuda',
max_norm=args.epsilon)
elif args.attack == 'DDN':
print('Attacker is DDN')
attacker = DDN(steps=args.num_steps,
device='cuda',
max_norm=args.epsilon,
init_norm=args.init_norm_DDN,
gamma=args.gamma_DDN)
else:
raise Exception('Unknown attack')
criterion = CrossEntropyLoss().cuda()
optimizer = SGD(filter(lambda p: p.requires_grad, model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = StepLR(optimizer,
step_size=args.lr_step_size,
gamma=args.gamma)
starting_epoch = 0
logfilename = os.path.join(args.outdir, 'log.txt')
# Load latest checkpoint if exists (to handle philly failures)
model_path = os.path.join(args.outdir, 'checkpoint.pth.tar')
if args.resume:
if os.path.isfile(model_path):
print("=> loading checkpoint '{}'".format(model_path))
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
starting_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
model_path, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(model_path))
if args.adv_training:
init_logfile(
logfilename,
"epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc\ttestaccNor"
)
else:
init_logfile(
logfilename,
"epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc")
else:
if args.adv_training:
init_logfile(
logfilename,
"epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc\ttestaccNor"
)
else:
init_logfile(
logfilename,
"epoch\ttime\tlr\ttrainloss\ttestloss\ttrainacc\ttestacc")
for epoch in range(starting_epoch, args.epochs):
scheduler.step(epoch)
attacker.max_norm = np.min(
[args.epsilon, (epoch + 1) * args.epsilon / args.warmup])
attacker.init_norm = np.min(
[args.epsilon, (epoch + 1) * args.epsilon / args.warmup])
before = time.time()
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, args.noise_sd,
attacker)
test_loss, test_acc, test_acc_normal = test(test_loader, model,
criterion, args.noise_sd,
attacker)
after = time.time()
if args.adv_training:
log(
logfilename,
"{}\t{:.2f}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format(
epoch, after - before,
scheduler.get_lr()[0], train_loss, test_loss, train_acc,
test_acc, test_acc_normal))
else:
log(
logfilename,
"{}\t{:.2f}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format(
epoch, after - before,
scheduler.get_lr()[0], train_loss, test_loss, train_acc,
test_acc))
torch.save(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, model_path)
def main(args):
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# load the base classifier
checkpoint = torch.load(args.base_classifier)
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
print('arch:', checkpoint['arch'])
if checkpoint["arch"] == 'resnet50' and args.dataset == "imagenet":
try:
base_classifier.load_state_dict(checkpoint['state_dict'])
except Exception as e:
print('direct load failed, try alternative')
try:
base_classifier = torchvision.models.resnet50(
pretrained=False).cuda()
base_classifier.load_state_dict(checkpoint['state_dict'])
# fix
# normalize_layer = get_normalize_layer('imagenet').cuda()
# base_classifier = torch.nn.Sequential(normalize_layer, base_classifier)
except Exception as e:
print('alternative failed again, try alternative 2')
base_classifier = torchvision.models.resnet50(
pretrained=False).cuda()
# base_classifier.load_state_dict(checkpoint['state_dict'])
normalize_layer = get_normalize_layer('imagenet').cuda()
base_classifier = torch.nn.Sequential(normalize_layer,
base_classifier)
base_classifier.load_state_dict(checkpoint['state_dict'])
else:
base_classifier.load_state_dict(checkpoint['state_dict'])
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
# generate transformer
transformer = gen_inference_transformer(args, dataset[0][0])
smoothed_classifier = SemanticSmooth(base_classifier,
get_num_classes(args.dataset),
transformer)
# generate image-level transform and params
tinst1, tfunc1, tinst2, tfunc2, param1l, param1r, param2l, param2r, candidates = gen_transform_and_params(
args, dataset[0][0])
# init random number generator
m1 = Uniform(param1l, param1r)
if param2l is not None:
m2 = Uniform(param2l, param2r)
# init metrics
tot = tot_benign = tot_robust = 0
# [main] attack section
for i in range(len(dataset)):
# only certify every args.skip examples
if i % args.skip != 0:
continue
print('working on #', i)
(x, y) = dataset[i]
# clean_x = x.cuda().unsqueeze(0)
pred = smoothed_classifier.predict(x, args.N0, args.p, args.batch)
if pred != y:
pass
else:
tot_benign += 1
robust = True
for j in range(0, args.tries):
xp = None
if args.transtype == 'translation':
param_sample1 = candidates[int(m1.sample().item())]
xp = tfunc1(tinst1, x, param_sample1[0].item(),
param_sample1[1].item())
else:
param_sample1 = m1.sample().item()
if param2l is not None:
param_sample2 = m2.sample().item()
xp = tfunc1(tinst1, x, param_sample1)
if param2l is not None:
xp = tfunc2(tinst2, xp, param_sample2)
# xp = xp.contiguous().cuda()
# xp_old = xp
# if args.l2 is not None and args.l2 > EPS:
# xp = fgsm(model, xp, torch.tensor([y], dtype=torch.long).expand(now_batch).cuda(), args.l2)
# print(torch.norm((xp_old - xp).reshape(xp.size()[0], -1), dim=1))
xp = xp.type_as(x)
if args.transtype in [
'rotation-brightness-l2', 'scaling-brightness-l2'
]:
# compute the gradient by soft label and empirical mean
smoothed_classifier.base_classifier.eval()
grad = torch.zeros(
(args.N0, xp.shape[0], xp.shape[1], xp.shape[2]))
n = 0
while n < args.N0:
now_batch = min(args.batch, args.N0 - n)
batch = xp.repeat((now_batch, 1, 1, 1))
batch_noised = smoothed_classifier.transformer.process(
batch).cuda()
batch_noised = Variable(
batch_noised.data,
requires_grad=True).contiguous()
opt = torch.optim.Adam([batch_noised], lr=1e-3)
opt.zero_grad()
loss = torch.nn.CrossEntropyLoss()(
smoothed_classifier.base_classifier(batch_noised),
torch.tensor(
[y],
dtype=torch.long).expand(now_batch).cuda())
loss.backward()
grad[n:n + now_batch, :, :, :] = batch_noised.grad.data
n += now_batch
grad = torch.mean(grad, dim=0)
unit_grad = F.normalize(grad,
p=2,
dim=list(range(grad.dim())))
delta = unit_grad * args.l2_r
# print(xp)
xp = xp + delta
# print(xp + delta)
# print(delta)
# print(torch.norm(delta.reshape(-1)))
pred = smoothed_classifier.predict(xp, args.N0, args.p,
args.batch)
if (pred != y):
robust = False
break
print(f"> {j}/{args.tries}", end='\r', flush=True)
tot_robust += int(robust)
tot += 1
print(
f'#{i} clean acc={tot_benign / tot} robust acc={tot_robust / tot}')
if args.outfile is None:
param_str = ''
if args.transtype != 'translation':
param_str = f'{param1r}'
if param2r is not None:
param_str += f'_{param2r}'
else:
param_str = f'{args.displacement}'
args.outfile = args.transtype + '/' + args.dataset + '/' + param_str + '/' + 'result.txt'
out_full_path = os.path.join(args.outfolder, args.outfile)
print('output result to ' + out_full_path)
if not os.path.exists(os.path.dirname(out_full_path)):
os.makedirs(os.path.dirname(out_full_path))
f = open(out_full_path, 'w')
f.write(
f'clean {tot_benign / tot},{tot_benign} robust={tot_robust / tot},{tot_robust} tot={tot}\n'
)
f.close()
print('done')
parser.add_argument("--N", type=int, default=100000,
help="number of samples to use")
parser.add_argument("--alpha", type=float, default=0.001,
help="failure probability")
args = parser.parse_args()
if __name__ == "__main__":
# load the base classifier
checkpoint = torch.load(args.base_classifier,
map_location=torch.device('cpu'))
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
base_classifier.load_state_dict(checkpoint['state_dict'])
# create the smooothed classifier g
smoothed_classifier = Smooth(
base_classifier, get_num_classes(args.dataset), args.sigma)
# prepare output file
f = open(args.outfile, 'w')
print("idx\tlabel\tpredict\tradius\tcorrect\ttime", file=f, flush=True)
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
for i in range(len(dataset)):
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i == args.max:
break
def prologue(args):
if not hasattr(args, 'id') or args.id is None:
args.id = np.random.randint(10000)
args.outdir = args.outdir + f"/{args.arch}/{args.id}/"
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
# Copies files to the outdir to store complete script with each experiment
copy_code(args.outdir)
train_dataset = get_dataset(args.dataset, 'train')
test_dataset = get_dataset(args.dataset, 'test')
pin_memory = (args.dataset == "imagenet")
train_loader = DataLoader(train_dataset,
shuffle=True,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
test_loader = DataLoader(test_dataset,
shuffle=False,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
if args.pretrained_model != '':
assert args.arch == 'cifar_resnet110', 'Unsupported architecture for pretraining'
checkpoint = torch.load(args.pretrained_model)
model = get_architecture(checkpoint["arch"], args.dataset)
model.load_state_dict(checkpoint['state_dict'])
model[1].fc = nn.Linear(64, get_num_classes('cifar10')).to(device)
else:
model = get_architecture(args.arch, args.dataset)
logfilename = os.path.join(args.outdir, 'log.txt')
init_logfile(logfilename,
"epoch\ttime\tlr\ttrain loss\ttrain acc\ttestloss\ttest acc")
writer = SummaryWriter(args.outdir)
criterion = CrossEntropyLoss().to(device)
optimizer = SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = StepLR(optimizer,
step_size=args.lr_step_size,
gamma=args.gamma)
starting_epoch = 0
# Load latest checkpoint if exists (to handle philly failures)
model_path = os.path.join(args.outdir, 'checkpoint.pth.tar')
if args.resume:
if os.path.isfile(model_path):
print("=> loading checkpoint '{}'".format(model_path))
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
starting_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
model_path, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(model_path))
return train_loader, test_loader, criterion, model, optimizer, scheduler, \
starting_epoch, logfilename, model_path, device, writer
dataset = get_dataset(args.dataset, args.split)
for i in range(len(dataset)):
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i == args.max:
break
(x, label) = dataset[i]
#Smooth the classifier with this sigma
if not args.fix_sig_smooth:
args.sigma = sigma_test[i].item()
print('sigma is: ', args.sigma)
smoothed_classifier = Smooth(model, get_num_classes(args.dataset),
args.sigma)
#Now you can use the same exac
before_time = time()
# certify the prediction of g around x
x = x.cuda()
prediction, radius = smoothed_classifier.certify(
x, args.N0, args.N, args.alpha, args.batch)
after_time = time()
correct = int(prediction == label)
print(radius)
time_elapsed = str(
datetime.timedelta(seconds=(after_time - before_time)))
print("{}\t{}\t{}\t{:.3}\t{}\t{:.3}\t{}".format(
i, label, prediction, radius, correct, args.sigma, time_elapsed),
file=f,
help="number of samples to use")
parser.add_argument("--alpha",
type=float,
default=0.001,
help="failure probability")
args = parser.parse_args()
if __name__ == "__main__":
# load the base classifier
checkpoint = torch.load(args.base_classifier)
base_classifier = get_architecture(checkpoint["arch"], args.dataset)
base_classifier.load_state_dict(checkpoint['state_dict'])
# create the smoothed classifier g
smoothed_classifier = Smooth(base_classifier,
get_num_classes(args.dataset), args.sigma)
# prepare output file
f = open(args.outfile, 'w')
# iterate through the dataset
dataset = get_dataset(args.dataset, args.split)
print("idx\tlabel\tpredict\tcorrect\tscore\ttime", flush=True)
print("idx\tlabel\tpredict\tcorrect\tscore\ttime", file=f, flush=True)
for i in range(len(dataset)):
# only certify every args.skip examples, and stop after args.max examples
if i % args.skip != 0:
continue
if i == args.max:
break
def main():
iter_time = 1500
clients = []
average_loss = []
test_accuracy_rate = []
D_in = datasets.get_num_features("lfw")
D_out = datasets.get_num_classes("lfw")
batch_size = 4
train_cut = 0.8
print("Creating clients")
for i in range(10):
model = returnModel(D_in, D_out)
clients.append(
Client("lfw", "lfw_maleness_train" + str(i), batch_size, model,
train_cut))
model = returnModel(D_in, D_out)
test_client = Client("lfw", "lfw_maleness_test", batch_size, model, 0)
print("Training for iterations")
for iter in range(iter_time):
# Calculate and aggregaate gradients
for i in range(10):
clients[0].updateGrad(clients[i].getGrad())
# Share updated model
clients[0].step()
modelWeights = clients[0].getModelWeights()
for i in range(10):
clients[i].updateModel(modelWeights)
# Print average loss across clients
if iter % 100 == 0:
loss = 0.0
for i in range(10):
loss += clients[i].getLoss()
print("Average loss is " + str(loss / len(clients)))
test_client.updateModel(modelWeights)
test_err = test_client.getTestErr()
print("Test error: " + str(test_err))
accuracy_rate = 1 - test_err
print("Accuracy rate: " + str(accuracy_rate) + "\n")
average_loss.append(loss / len(clients))
test_accuracy_rate.append(accuracy_rate)
# plot average loss and accuracy rate of the updating model
x = range(1, int(math.floor(iter_time / 100)) + 1)
fig, ax1 = plt.subplots()
ax1.plot(x,
average_loss,
color='orangered',
label='lfw_gender_average_loss')
plt.legend(loc=2)
ax2 = ax1.twinx()
ax2.plot(x,
test_accuracy_rate,
color='blue',
label='lfw_gender_test_accuracy_rate')
plt.legend(loc=1)
ax1.set_xlabel("iteration time / 100")
ax1.set_ylabel("average_loss")
ax2.set_ylabel("accuracy_rate")
plt.title("lfw_gender_graph")
plt.legend()
mp.show()
test_client.updateModel(modelWeights)
test_err = test_client.getTestErr()
print("Test error: " + str(test_err))
accuracy_rate = 1 - test_err
print("Accuracy rate: " + str(accuracy_rate) + "\n")
