def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
batch_size = 200
model_location = '{}/{}/model/{}_raw.pt'.format('../RawModels',
args.dataset, args.dataset)
# Get training parameters, set up model frameworks and then get the train_loader and test_loader
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
if dataset == 'MNIST':
raw_model = MNISTConvNet().to(device)
test_loader = get_mnist_test_loader(dir_name='../RawModels/MNIST/',
batch_size=batch_size)
else:
raw_model = resnet20_cifar().to(device)
test_loader = get_cifar10_test_loader(dir_name='../RawModels/CIFAR10/',
batch_size=batch_size)
raw_model.load(path=model_location, device=device)
defense_name = 'RT'
rt = RTDefense(model=raw_model,
defense_name=defense_name,
dataset=dataset,
device=device)
# predicting the testing dataset using the randomization transformation defense
raw_model.eval()
total = 0.0
correct = 0.0
with torch.no_grad():
for index, (images, labels) in enumerate(test_loader):
# input images first go through the randomization transformation layer and then the resulting images are feed into the original model
transformed_images = rt.randomization_transformation(
samples=images,
original_size=images.shape[-1],
final_size=args.resize)
outputs = raw_model(transformed_images)
labels = labels.to(device)
_, predicted = torch.max(outputs.data, 1)
total = total + labels.size(0)
correct = correct + (predicted == labels).sum().item()
ratio = correct / total
print(
'\nTest accuracy of the {} model on the testing dataset: {:.1f}/{:.1f} = {:.2f}%\n'
.format(raw_model.model_name, correct, total, ratio * 100))
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
# load the raw model / testing dataset loader
raw_model_location = '{}{}/model/{}_raw.pt'.format('../RawModels/', dataset, dataset)
if dataset == 'MNIST':
raw_model = MNISTConvNet().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name='../RawModels/MNIST/', batch_size=30)
else:
raw_model = resnet20_cifar().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name='../RawModels/CIFAR10/', batch_size=25)
raw_model.eval()
# get predictions of the raw model on test datasets
predicted_raw, true_label = prediction(model=raw_model, test_loader=test_loader, device=device)
re_train_defenses = {'NAT', 'EAT', 'PAT', 'DD', 'IGR'}
input_transformation_defenses = {'EIT', 'RT', 'PD', 'TE'}
other_defenses = {'RC'}
defense_name = args.defense.upper().strip()
if defense_name in re_train_defenses:
print('\nthe ##{}## defense is a kind of complete defenses that retrain the model'.format(defense_name))
# load the defense-enhanced model
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format('../DefenseEnhancedModels', defense_name, dataset, defense_name)
defended_model = MNISTConvNet().to(device) if dataset == 'MNIST' else resnet20_cifar().to(device)
defended_model.load(path=defended_model_location, device=device)
defended_model.eval()
predicted_defended, _ = prediction(model=defended_model, test_loader=test_loader, device=device)
raw_acc, def_acc, cav, crr, csr, ccv, cos = defense_utility_measure(predicted_defended, predicted_raw, true_label)
elif defense_name in input_transformation_defenses:
print('\nthe ##{}## defense is a kind of complete defense hat need to transform the images ... '.format(defense_name))
if defense_name == 'EIT':
from Defenses.DefenseMethods.EIT import EITDefense, TransformedDataset
eit_params = {
'crop_size': args.crop_size,
'lambda_tv': args.lambda_tv,
'JPEG_quality': args.JPEG_quality,
'bit_depth': args.bit_depth
}
defended_model = MNISTConvNet().to(device) if dataset == 'MNIST' else resnet20_cifar().to(device)
EIT = EITDefense(model=defended_model, defense_name=defense_name, dataset=dataset, re_training=False, training_parameters=None,
device=device, **eit_params)
transformed_test_data_numpy, test_label_numpy = EIT.transforming_dataset(data_loader=test_loader)
transformed_test_dataset = TransformedDataset(images=torch.from_numpy(transformed_test_data_numpy),
labels=torch.from_numpy(test_label_numpy), dataset=dataset, transform=None)
transformed_test_loader = torch.utils.data.DataLoader(transformed_test_dataset, batch_size=100, shuffle=False)
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format('../DefenseEnhancedModels', defense_name, dataset, defense_name)
defended_model = defended_model.to(device)
defended_model.load(path=defended_model_location, device=device)
defended_model.eval()
predicted_defended, _ = prediction(model=defended_model, test_loader=transformed_test_loader, device=device)
raw_acc, def_acc, cav, crr, csr, ccv, cos = defense_utility_measure(predicted_defended, predicted_raw, true_label)
elif defense_name == 'RT':
final_size = args.resize
assert isinstance(final_size, int)
warnings.warn(message='For the RT defense, the #resize# parameter is specified as {}, please check ...'.format(final_size))
predicted_defended = rt_prediction(model=raw_model, dataset=dataset, data_loader=test_loader, final_size=final_size, device=device)
# test the utility performance of defended model
raw_acc, def_acc, cav, crr, csr, ccv, cos = defense_utility_measure(predicted_defended, predicted_raw, true_label)
elif defense_name == 'PD':
epsilon = args.epsilon
warnings.warn(message='For the PixelDefend defense, the #epsilon# parameter is specified as {}, please check ...'.format(epsilon))
predicted_defended = pd_prediction(model=raw_model, dataset=dataset, data_loader=test_loader, epsilon=epsilon, device=device)
raw_acc, def_acc, cav, crr, csr, ccv, cos = defense_utility_measure(predicted_defended, predicted_raw, true_label)
else:
level = args.level
assert defense_name == 'TE' and isinstance(level, int)
warnings.warn(message='For the TE defense, the #level# parameter is specified as {}, please check ...'.format(level))
# load the defense-enhanced model (for TE)
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format('../DefenseEnhancedModels', defense_name, dataset, defense_name)
defended_model = MNISTConvNet(thermometer=True, level=level) if dataset == 'MNIST' else resnet20_cifar(thermometer=True, level=level)
defended_model = defended_model.to(device)
defended_model.load(path=defended_model_location, device=device)
defended_model.eval()
predicted_defended = te_prediction(model=defended_model, data_loader=test_loader, level=level, device=device)
raw_acc, def_acc, cav, crr, csr, ccv, cos = defense_utility_measure(predicted_defended, predicted_raw, true_label)
else:
if defense_name == 'RC':
print('\n##{}## defense is a kind of region-based classification defenses ... '.format(defense_name))
from Defenses.DefenseMethods.RC import RCDefense
num_points = 1000
radius = args.radius
rc = RCDefense(model=raw_model, defense_name='RC', dataset=dataset, device=device, num_points=num_points)
predicted_defended = []
with torch.no_grad():
for index, (images, labels) in enumerate(test_loader):
rc_labels = rc.region_based_classification(samples=images, radius=radius)
predicted_defended.extend(rc_labels)
predicted_defended = np.array(predicted_defended)
# classification accuracy of defense-enhanced model
correct_prediction_def = np.equal(predicted_defended, true_label)
def_acc = np.mean(correct_prediction_def.astype(float))
# classification accuracy of raw model
correct_prediction_raw = np.equal(np.argmax(predicted_raw, axis=1), true_label)
raw_acc = np.mean(correct_prediction_raw.astype(float))
# Classification Accuracy Variance(CAV)
cav = def_acc - raw_acc
# Find the index of correct predicted examples by defence-enhanced model and raw model
idx_def = np.squeeze(np.argwhere(correct_prediction_def == True))
idx_raw = np.squeeze(np.argwhere(correct_prediction_raw == True))
idx = np.intersect1d(idx_def, idx_raw, assume_unique=True)
# Compute the Classification Rectify Ratio(CRR) & Classification Sacrifice Ratio(CSR)
crr = (len(idx_def) - len(idx)) / len(predicted_raw)
csr = (len(idx_raw) - len(idx)) / len(predicted_raw)
ccv = cos = 0
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
print("****************************")
print("The utility evaluation results of the {} defense for {} Dataset are as follow:".format(defense_name, dataset))
print('Acc of Raw Model:\t\t{:.2f}%'.format(raw_acc * 100))
print('Acc of {}-enhanced Model:\t{:.2f}%'.format(defense_name, def_acc * 100))
print('CAV: {:.2f}%'.format(cav * 100))
print('CRR: {:.2f}%'.format(crr * 100))
print('CSR: {:.2f}%'.format(csr * 100))
print('CCV: {:.2f}%'.format(ccv * 100))
print('COS: {:.4f}'.format(cos))
print("****************************")
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# get the training and testing dataset loaders
train_loader, valid_loader = get_mnist_train_validate_loader(
dir_name='./MNIST/',
batch_size=MNIST_Training_Parameters['batch_size'],
valid_size=0.1,
shuffle=True)
test_loader = get_mnist_test_loader(
dir_name='./MNIST/',
batch_size=MNIST_Training_Parameters['batch_size'])
# set up the model and optimizer
mnist_model = MNISTConvNet().to(device)
optimizer = optim.SGD(mnist_model.parameters(),
lr=MNIST_Training_Parameters['learning_rate'],
momentum=MNIST_Training_Parameters['momentum'],
weight_decay=MNIST_Training_Parameters['decay'],
nesterov=True)
# Training
best_val_acc = None
model_saver = './MNIST/model/MNIST_' + 'raw' + '.pt'
for epoch in range(MNIST_Training_Parameters['num_epochs']):
# training the model within one epoch
train_one_epoch(model=mnist_model,
train_loader=train_loader,
optimizer=optimizer,
epoch=epoch,
device=device)
# validation
val_acc = validation_evaluation(model=mnist_model,
validation_loader=valid_loader,
device=device)
if not best_val_acc or round(val_acc, 4) >= round(best_val_acc, 4):
if best_val_acc is not None:
os.remove(model_saver)
best_val_acc = val_acc
mnist_model.save(name=model_saver)
else:
print(
'Train Epoch{:>3}: validation dataset accuracy did not improve from {:.4f}\n'
.format(epoch, best_val_acc))
# Testing
final_model = copy.deepcopy(mnist_model)
final_model.load(path=model_saver, device=device)
accuracy = testing_evaluation(model=final_model,
test_loader=test_loader,
device=device)
print(
'Finally, the ACCURACY of saved model [{}] on testing dataset is {:.2f}%\n'
.format(final_model.model_name, accuracy * 100.0))
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# Get training parameters, set up model frames and then get the train_loader and test_loader
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
if dataset == 'MNIST':
training_parameters = MNIST_Training_Parameters
model_framework = MNISTConvNet().to(device)
batch_size = training_parameters['batch_size']
# train_loader for training the initial model
train_loader, valid_loader = get_mnist_train_validate_loader(
dir_name='../RawModels/MNIST/',
batch_size=batch_size,
valid_size=0.1,
shuffle=True)
# raw train_loader (no augmentation) for constructing the SoftLabelDataset and then used to train the distilled model
raw_train_loader, raw_valid_loader = get_mnist_train_validate_loader(
dir_name='../RawModels/MNIST/',
batch_size=batch_size,
valid_size=0.1,
shuffle=False)
# testing dataset loader
test_loader = get_mnist_test_loader(dir_name='../RawModels/MNIST/',
batch_size=batch_size)
else:
training_parameters = CIFAR10_Training_Parameters
model_framework = resnet20_cifar().to(device)
batch_size = training_parameters['batch_size']
# train_loader for training the initial model
train_loader, valid_loader = get_cifar10_train_validate_loader(
dir_name='../RawModels/CIFAR10/',
batch_size=batch_size,
valid_size=0.1,
augment=True,
shuffle=True)
# raw train_loader (no augmentation) for constructing the SoftLabelDataset and then used to train the distilled model
raw_train_loader, raw_valid_loader = get_cifar10_train_validate_loader(
dir_name='../RawModels/CIFAR10/',
batch_size=batch_size,
valid_size=0.1,
augment=False,
shuffle=False)
# testing dataset loader
test_loader = get_cifar10_test_loader(dir_name='../RawModels/CIFAR10/',
batch_size=batch_size)
defense_name = 'DD'
dd = DistillationDefense(model=model_framework,
defense_name=defense_name,
dataset=dataset,
temperature=args.temp,
training_parameters=training_parameters,
device=device)
dd.defense(initial_flag=args.initial,
train_loader=train_loader,
validation_loader=valid_loader,
raw_train=raw_train_loader,
raw_valid=raw_valid_loader,
test_loader=test_loader)
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# Get training parameters, set up model frameworks and then get the train_loader and test_loader
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
if dataset == 'MNIST':
training_parameters = MNIST_Training_Parameters
model_framework = MNISTConvNet().to(device)
batch_size = training_parameters['batch_size']
train_loader, valid_loader = get_mnist_train_validate_loader(
dir_name='../RawModels/MNIST/',
batch_size=batch_size,
valid_size=0.1,
shuffle=True)
test_loader = get_mnist_test_loader(dir_name='../RawModels/MNIST/',
batch_size=batch_size)
else:
training_parameters = CIFAR10_Training_Parameters
model_framework = resnet20_cifar().to(device)
batch_size = training_parameters['batch_size']
train_loader, valid_loader = get_cifar10_train_validate_loader(
dir_name='../RawModels/CIFAR10/',
batch_size=batch_size,
valid_size=0.1,
shuffle=True)
test_loader = get_cifar10_test_loader(dir_name='../RawModels/CIFAR10/',
batch_size=batch_size)
defense_name = 'EAT'
eat_params = {'eps': args.eps, 'alpha': args.alpha}
eat = EATDefense(model=model_framework,
defense_name=defense_name,
dataset=dataset,
training_parameters=training_parameters,
device=device,
**eat_params)
# train the external models
if args.train_externals:
print('\nStart to train the external models ......\n')
eat.train_external_model_group(train_loader=train_loader,
validation_loader=valid_loader)
# load the external models
pre_train_models = eat.load_external_model_group(
model_dir='../DefenseEnhancedModels/EAT/', test_loader=test_loader)
eat.defense(pre_trained_models=pre_train_models,
train_loader=train_loader,
validation_loader=valid_loader)
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# prepare the dataset name, candidate num, dataset location and raw model location
dataset = args.dataset.upper()
num = args.number
dataset_location = '../RawModels/{}/'.format(dataset)
raw_model_location = '../RawModels/{}/model/{}_raw.pt'.format(
dataset, dataset)
print(
"\nStarting to select {} {} Candidates Example, which are correctly classified by the Raw Model from {}\n"
.format(num, dataset, raw_model_location))
# load the raw model and testing dataset
assert args.dataset == 'MNIST' or args.dataset == 'CIFAR10'
if dataset == 'MNIST':
raw_model = MNISTConvNet().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
else:
raw_model = resnet20_cifar().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
# get the successfully classified examples
successful = []
raw_model.eval()
with torch.no_grad():
for image, label in test_loader:
image = image.to(device)
label = label.to(device)
output = raw_model(image)
_, predicted = torch.max(output.data, 1)
if predicted == label:
_, least_likely_class = torch.min(output.data, 1)
successful.append([image, label, least_likely_class])
print(len(successful))
candidates = random.sample(successful, num)
candidate_images = []
candidate_labels = []
candidates_llc = []
candidate_targets = []
for index in range(len(candidates)):
image = candidates[index][0].cpu().numpy()
image = np.squeeze(image, axis=0)
candidate_images.append(image)
label = candidates[index][1].cpu().numpy()[0]
llc = candidates[index][2].cpu().numpy()[0]
# selection for the targeted label
classes = [i for i in range(10)]
classes.remove(label)
target = random.sample(classes, 1)[0]
one_hot_label = [0 for i in range(10)]
one_hot_label[label] = 1
one_hot_llc = [0 for i in range(10)]
one_hot_llc[llc] = 1
one_hot_target = [0 for i in range(10)]
one_hot_target[target] = 1
candidate_labels.append(one_hot_label)
candidates_llc.append(one_hot_llc)
candidate_targets.append(one_hot_target)
candidate_images = np.array(candidate_images)
candidate_labels = np.array(candidate_labels)
candidates_llc = np.array(candidates_llc)
candidate_targets = np.array(candidate_targets)
if dataset not in os.listdir('./'):
os.mkdir('./{}/'.format(dataset))
else:
shutil.rmtree('{}'.format(dataset))
os.mkdir('./{}/'.format(dataset))
np.save('./{}/{}_inputs.npy'.format(dataset, dataset), candidate_images)
np.save('./{}/{}_labels.npy'.format(dataset, dataset), candidate_labels)
np.save('./{}/{}_llc.npy'.format(dataset, dataset), candidates_llc)
np.save('./{}/{}_targets.npy'.format(dataset, dataset), candidate_targets)
def main(args):
# Device configuration
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
batch_size = 1000
# get the training_parameters, raw_model, and test_loader
model_location = '{}/{}/model/{}_raw.pt'.format('../RawModels', dataset, dataset)
if dataset == 'MNIST':
raw_model = MNISTConvNet().to(device)
test_loader = get_mnist_test_loader(dir_name='../RawModels/MNIST/', batch_size=batch_size)
else:
raw_model = resnet20_cifar().to(device)
test_loader = get_cifar10_test_loader(dir_name='../RawModels/CIFAR10/', batch_size=batch_size)
raw_model.load(path=model_location, device=device)
defense_name = 'RC'
rc = RCDefense(model=raw_model, defense_name=defense_name, dataset=dataset, device=device, num_points=args.num_points)
# search the radius r
if args.search:
# get the validation dataset (10% with the training dataset)
print('start to search the radius r using validation dataset ...')
if dataset == 'MNIST':
_, valid_loader = get_mnist_train_validate_loader(dir_name='../RawModels/MNIST/', batch_size=batch_size, valid_size=0.02,
shuffle=True)
else:
_, valid_loader = get_cifar10_train_validate_loader(dir_name='../RawModels/CIFAR10/', batch_size=batch_size, valid_size=0.02,
shuffle=True)
radius = rc.search_best_radius(validation_loader=valid_loader, radius_min=args.radius_min, radius_max=args.radius_max,
radius_step=args.radius_step)
else:
radius = round(args.radius, 2)
print('######\nthe radius for RC is set or searched as: {}\n######'.format(radius))
# calculate the accuracy of region-based classification defense on testing dataset
print('\nStart to calculate the accuracy of region-based classification defense on testing dataset')
raw_model.eval()
total = 0.0
correct = 0.0
with torch.no_grad():
for images, labels in test_loader:
# apply the region-based classification on images using radius
rc_labels = rc.region_based_classification(samples=images, radius=radius)
rc_labels = torch.from_numpy(rc_labels)
total += labels.size(0)
correct += (rc_labels == labels).sum().item()
ratio = correct / total
print('\nTest accuracy of the {} model on the testing dataset: {:.1f}/{:.1f} = {:.2f}%\n'.format(raw_model.model_name, correct, total,
ratio * 100))