# Hyper-parameters
param = {
'test_batch_size': 100,
'epsilon': 0.3,
}
# Data loaders
test_dataset = datasets.MNIST(root='datasets/MNIST_data', train=True, download=True,
transform=transforms.ToTensor())
loader_test = torch.utils.data.DataLoader(test_dataset,
batch_size=param['test_batch_size'], shuffle=False)
# Setup model to be attacked
net = LeNet5()
net.load_state_dict(torch.load('models/adv_trained_lenet5.pkl'))
if torch.cuda.is_available():
print('CUDA ensabled.')
net.cuda()
for p in net.parameters():
p.requires_grad = False
net.eval()
test(net, loader_test)
# Adversarial attack
adversary = FGSMAttack(net, param['epsilon'])
valid_dataset = CustomDataset(valid_img, valid_label, transform['valid'])
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=para['batch_size'],
shuffle=False,
num_workers=para['workers'],
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset,
batch_size=para['batch_size'],
shuffle=False,
num_workers=para['workers'],
pin_memory=True)
#%% Build a model
# net = VGG(dataset=para['dataset'], pretrained=para['pretrain'])
net = LeNet5('aoi')
# Send model into gpu memory
if para['cuda']:
net.cuda()
log.log('Model Structure:\n{}'.format(net))
#%% Create loss function, optimzier and training scheduler
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=para['lr'],
weight_decay=para['decay'],
momentum=0.9,
nesterov=True)
def MNIST_bbox_sub(param, loader_hold_out, loader_test):
"""
Train a substitute model using Jacobian data augmentation
arXiv:1602.02697
"""
# Setup the substitute
net = SubstituteModel()
if torch.cuda.is_available():
print('CUDA ensabled for the substitute.')
net.cuda()
net.train()
# Setup the oracle
oracle = LeNet5()
if torch.cuda.is_available():
print('CUDA ensabled for the oracle.')
oracle.cuda()
oracle.load_state_dict(torch.load(param['oracle_name'] + '.pkl'))
oracle.eval()
# Setup training
criterion = nn.CrossEntropyLoss()
# Careful optimization is crucial to train a well-representative
# substitute. In Tensorflow Adam has some problem:
# (https://github.com/tensorflow/cleverhans/issues/183)
# But it works fine here in PyTorch (you may try other optimization
# methods
optimizer = torch.optim.Adam(net.parameters(), lr=param['learning_rate'])
# Data held out for initial training
data_iter = iter(loader_hold_out)
X_sub, y_sub = data_iter.next()
X_sub, y_sub = X_sub.numpy(), y_sub.numpy()
# Train the substitute and augment dataset alternatively
for rho in range(param['data_aug']):
print("Substitute training epoch #" + str(rho))
print("Training data: " + str(len(X_sub)))
rng = np.random.RandomState()
# model training
for epoch in range(param['nb_epochs']):
print('Starting epoch %d / %d' % (epoch + 1, param['nb_epochs']))
# Compute number of batches
nb_batches = int(
np.ceil(float(len(X_sub)) / param['test_batch_size']))
assert nb_batches * param['test_batch_size'] >= len(X_sub)
# Indices to shuffle training set
index_shuf = list(range(len(X_sub)))
rng.shuffle(index_shuf)
for batch in range(nb_batches):
# Compute batch start and end indices
start, end = batch_indices(batch, len(X_sub),
param['test_batch_size'])
x = X_sub[index_shuf[start:end]]
y = y_sub[index_shuf[start:end]]
scores = net(to_var(torch.from_numpy(x)))
loss = criterion(scores, to_var(torch.from_numpy(y).long()))
optimizer.zero_grad()
loss.backward()
optimizer.step()
print('loss = %.8f' % (loss.data[0]))
test(net,
loader_test,
blackbox=True,
hold_out_size=param['hold_out_size'])
# If we are not at last substitute training iteration, augment dataset
if rho < param['data_aug'] - 1:
print("Augmenting substitute training data.")
# Perform the Jacobian augmentation
X_sub = jacobian_augmentation(net, X_sub, y_sub)
print("Labeling substitute training data.")
# Label the newly generated synthetic points using the black-box
scores = oracle(to_var(torch.from_numpy(X_sub)))
# Note here that we take the argmax because the adversary
# only has access to the label (not the probabilities) output
# by the black-box model
y_sub = np.argmax(scores.data.cpu().numpy(), axis=1)
torch.save(net.state_dict(), param['oracle_name'] + '_sub.pkl')
hold_out_data,
batch_size=param['hold_out_size'],
sampler=hold_out_sampler,
shuffle=False)
loader_test = torch.utils.data.DataLoader(
test_dataset,
batch_size=param['test_batch_size'],
sampler=test_sampler,
shuffle=False)
# Train the substitute
MNIST_bbox_sub(param, loader_hold_out, loader_test)
# Setup models
net = SubstituteModel()
oracle = LeNet5()
net.load_state_dict(torch.load(param['oracle_name'] + '_sub.pkl'))
oracle.load_state_dict(torch.load(param['oracle_name'] + '.pkl'))
if torch.cuda.is_available():
net.cuda()
oracle.cuda()
print('CUDA ensabled.')
for p in net.parameters():
p.requires_grad = False
net.eval()
oracle.eval()
from models import LeNet5
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
''' =============== Learning Setting =============== '''
exp_name = 'LeNet5_train1'
CONTINUE_LEARNING = False
train_ratio = 0.8
train_batch_size, test_batch_size = 128, 128
epochs = 3
save_period = 5
learning_rate = 0.01
model = LeNet5()
optimizer = SGD(learning_rate=learning_rate)
''' =============== Learning Setting =============== '''
loss_object = SparseCategoricalCrossentropy()
path_dict = dir_setting(exp_name, CONTINUE_LEARNING)
model, losses_accs, start_epoch = continue_setting(CONTINUE_LEARNING,
path_dict,
model=model)
train_ds, validation_ds, test_ds = load_processing_mnist(
train_ratio, train_batch_size, test_batch_size)
metric_objects = get_classification_metrics()
for epoch in range(start_epoch, epochs):
train(train_ds, model, loss_object, optimizer, metric_objects)
validation(validation_ds, model, loss_object, metric_objects)
def main(args):
set_seed(SEED)
train_transforms, test_transforms = get_transforms(args.dataset)
print(f"Data transformations:\n{train_transforms}\n")
# Get the dataloaders
train_loader, test_loader = get_dataloaders(args.dataset, args.batch_size,
args.workers, train_transforms,
test_transforms)
# Architecture
if args.dataset == 'mnist':
in_channels = 1
else:
raise NotImplementedError()
if args.activation == 'relu':
activation = nn.ReLU(inplace=True)
else:
raise NotImplementedError()
if args.pooling == 'max':
pooling = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
else:
raise NotImplementedError()
drop_rate = args.drop_rate
# Build model
model = LeNet5(in_channels, activation, pooling, drop_rate)
if torch.cuda.is_available():
torch.cuda.set_device(args.gpu)
model = model.cuda()
# Weight normal initialization
if args.init_weights:
model.apply(normal_initialization)
start_epoch = 0
if args.resume is not None:
model, optimizer, start_epoch = load_training_state(
model, optimizer, args.resume)
# Loss function & optimizer
if args.criterion == 'ce':
criterion = nn.CrossEntropyLoss()
else:
raise NotImplementedError()
if args.optimizer == 'sgd':
# Issue
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=args.nesterov)
elif args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
else:
raise NotImplementedError()
scheduler = ReduceLROnPlateau(optimizer,
factor=0.5,
patience=0,
threshold=1e-2,
verbose=True)
# Output folder
output_folder = os.path.join(args.output_folder, args.training_name)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
log_path = os.path.join(args.output_folder, 'logs', args.training_name)
if os.path.exists(log_path):
rmtree(log_path)
logger = SummaryWriter(log_path)
# Train
best_loss = math.inf
mb = master_bar(range(args.nb_epochs))
for epoch_idx in mb:
# Training
train_epoch(model,
train_loader,
optimizer,
criterion,
mb,
tb_logger=logger,
epoch=start_epoch + epoch_idx)
# Evaluation
val_loss, accuracy = evaluate(model, test_loader, criterion)
mb.first_bar.comment = f"Epoch {start_epoch+epoch_idx+1}/{start_epoch+args.nb_epochs}"
mb.write(
f'Epoch {start_epoch+epoch_idx+1}/{start_epoch+args.nb_epochs} - Validation loss: {val_loss:.4} (Acc@1: {accuracy:.2%})'
)
# State saving
if val_loss < best_loss:
print(
f"Validation loss decreased {best_loss:.4} --> {val_loss:.4}: saving state..."
)
best_loss = val_loss
torch.save(
dict(epoch=start_epoch + epoch_idx,
model_state_dict=model.state_dict(),
optimizer_state_dict=optimizer.state_dict(),
val_loss=val_loss),
os.path.join(output_folder, "training_state.pth"))
if logger is not None:
current_iter = (start_epoch + epoch_idx + 1) * len(train_loader)
logger.add_scalar(f"Validation loss", val_loss, current_iter)
logger.add_scalar(f"Error rate", 1 - accuracy, current_iter)
logger.flush()
scheduler.step(val_loss)
def train_lenet(**kwargs):
if kwargs['tensorboard']:
name, directory = set_directory(name=kwargs['name'],
type_net=kwargs['type_net'],
dof=kwargs['dof'])
writer = SummaryWriter(directory)
else:
writer = None
train_loader, val_loader, iter_per_epoch = load_mnist(
batch_size=kwargs['batch_size'])
model = LeNet5(num_classes=10,
type_net=kwargs['type_net'],
N=60000,
beta_ema=kwargs['beta_ema'],
dof=kwargs['dof'])
num_parameters = sum([p.data.nelement() for p in model.parameters()])
print(f'Number of model parameters: {num_parameters}')
if torch.cuda.is_available():
torch.cuda.set_device(kwargs['device'])
# for training on multiple GPUs.
# Use CUDA_VISIBLE_DEVICES=0,1 to specify which GPUs to use
if kwargs['multi_gpu']:
model = torch.nn.DataParallel(model).cuda()
else:
if torch.cuda.is_available():
model = model.cuda()
optimizer = construct_optimizer(optimizer=kwargs['optim'],
model=model,
lr=kwargs['lr'])
if kwargs['resume'] != '':
kwargs[
'start_epoch'], best_prec1, total_steps, model, optimizer = resume_from_checkpoint(
resume_path=kwargs['resume'], model=model, optimizer=optimizer)
else:
total_steps = 0
best_prec1 = 0.
cudnn.benchmark = True
loss_function = CrossEntropyLossWithAnnealing(
iter_per_epoch=iter_per_epoch,
total_steps=total_steps,
anneal_type=kwargs['anneal_type'],
anneal_kl=kwargs['anneal_kl'],
epzero=kwargs['epzero'],
epmax=kwargs['epmax'],
anneal_maxval=kwargs['anneal_maxval'],
writer=writer)
for epoch in range(kwargs['start_epoch'], kwargs['epochs']):
total_steps = train_single_epoch(train_loader=train_loader,
model=model,
criterion=loss_function,
optimizer=optimizer,
epoch=epoch,
clip_var=kwargs['clip_var'],
total_steps=total_steps,
print_freq=kwargs['print_freq'],
writer=writer,
thres_stds=kwargs['thres_std'])
prec1 = validate(val_loader=val_loader,
model=model,
criterion=loss_function,
epoch=epoch,
print_freq=kwargs['print_freq'],
writer=writer)
if kwargs['restart'] and epoch % kwargs['restart_interval'] == 0:
print('Restarting optimizer...')
optimizer = construct_optimizer(optimizer=kwargs['restart_optim'],
model=model,
lr=kwargs['restart_lr'])
is_best = prec1 > best_prec1
if is_best:
best_prec1 = prec1
if isinstance(model, torch.nn.DataParallel):
state = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': max(prec1, best_prec1),
'beta_ema': model.module.beta_ema,
'optimizer': optimizer.state_dict(),
'total_steps': total_steps
}
if model.module.beta_ema > 0:
state['avg_params'] = model.module.avg_param
state['steps_ema'] = model.module.steps_ema
else:
state = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': max(prec1, best_prec1),
'beta_ema': model.beta_ema,
'optimizer': optimizer.state_dict(),
'total_steps': total_steps
}
if model.beta_ema > 0:
state['avg_params'] = model.avg_param
state['steps_ema'] = model.steps_ema
if epoch in kwargs['save_at']:
name = f'checkpoint_{epoch}.pth.tar'
else:
name = 'checkpoint.pth.tar'
save_checkpoint(state=state, is_best=is_best, name=name)
print('Best accuracy: ', best_prec1)
if writer is not None:
writer.close()