def __init__(self, classifier, device='cuda'):
super(OddsAreOdd, self).__init__()
self.classifier = classifier
self.device = device
noise_eps = 'n0.01,s0.01,u0.01,n0.02,s0.02,u0.02,s0.03,n0.03,u0.03'
noise_eps_detect = 'n0.003,s0.003,u0.003,n0.005,s0.005,u0.005,s0.008,n0.008,u0.008'
_, train_loader, test_loader, _, _, _ = get_loaders('cifar10')
train_samples = get_data_tensor(train_loader, num=2000, offset=0, device='cuda')
with torch.no_grad():
self.predictor = collect_statistics(train_samples[0].cpu().numpy(), train_samples[1].cpu().numpy(),
latent_and_logits_fn_th=self.classifier.latent_and_logits,
nb_classes=10, weights=list(self.classifier.classifier.children())[-1].weight,
noise_eps=noise_eps.split(','), noise_eps_detect=noise_eps_detect.split(','))
next(self.predictor)
def progression_comparison(algoFlag, f_net, epsilon, norm, ntrials, nbase, search_space, algo='tpe', tau=4, sample_mult=2
, timelimit=1., use_vertex_removal=False, dataset='cifar10', fn='trials'):
"""
Main method to do the attack parameter search
"""
device = 'cuda'
_, _, test_loader, _, _, _ = get_loaders(dataset, 128, 128)
# model and threat model
rawdata = get_data_tensor(test_loader, num=10000, offset=0, device=device)
# 1. get network property
defense_property = defense_property_test(f_net.instance.model, test_loader)
# 2. network processing strategy
g_net = search_g_net(norm, epsilon, f_net, test_loader, defense_property, use_vertex_removal=use_vertex_removal)
filter_attack_list = [NoAttack()]
data = get_robust_img(filter_attack_list, f_net.instance, g_net.instance.model, rawdata, epsilon, norm)
global SEARCH_RESULT_LIST
SEARCH_RESULT_LIST = []
# 3. define search space
config = dict(g_net=g_net, f_net=f_net, defense_property=defense_property, epsilon=epsilon, norm=norm, repeat=1, timelimit=timelimit)
points_to_evaluate = None
if algoFlag:
print("Algorithm comparison mode: TPE vs Random")
# compare tpe and random
algos = ['tpe', 'random']
# use input search space
spaces = search_space.define_space(config, defense_property)
else:
print("Search space comparison mode: Loss vs no loss")
# use input algorithm
algos = algo
# compare search space with loss config and without loss config
search_space1 = Linf_search_space({"APGD": True})
search_space2 = Linf_search_space({"APGD": True, "use_loss": False})
space1 = search_space1.define_space(config, defense_property)
space2 = search_space2.define_space(config, defense_property)
spaces = [space1, space2]
# 4. start search in a sequential manner
get_trial_statistics(data, spaces, config, ntrials, nbase, points_to_evaluate=points_to_evaluate,
algos=algos, tau=tau, sample_mult=sample_mult, device=device, fn=fn)
return None
def attack_eval(attack,
f_net,
epsilon,
norm,
ntest,
use_vertex_removal,
dataset='cifar10'):
"""
Evaluate attack with network network transformation applied
"""
_, _, test_loader, _, _, _ = get_loaders(dataset, 128, 128)
defense_property = defense_property_test(f_net.instance.model, test_loader)
g_net = search_g_net(norm, epsilon, f_net, test_loader, defense_property,
use_vertex_removal)
return get_result_from_loader(attack,
g_net,
test_loader,
epsilon,
ntest,
norm,
eval_net=f_net)
from attack.attack_base import TargetedAttack
from zoo.cifar_model_pool import *
from utils.loader import get_loaders
from attack.network_transformation import *
from attack.attack_pool import *
import eagerpy as ep
device = 'cuda'
eps = 4/255
norm = ep.inf
ntest = 1000
# dataset
bounds = (0, 1)
trainN, train_loader, test_loader, input_size, input_channels, n_class = get_loaders('cifar10', 128, 128)
x_test, y_test = get_data_tensor(test_loader, ntest, device=device)
# meta attack scheme
# attack = meta_attack.baseline_attack
# attack = LinfProjectedGradientDescentAttack()
# attack.params["EOT"] = 1
untargeted_attack = LinfProjectedGradientDescentAttack()
attack = TargetedAttack(untargeted_attack)
model_class_list = [Model7]
for model_class in model_class_list:
model_test = model_class(load=False, device=device)
if not model_test.exists():
model_test.train(train_loader, eps, save=True)
custom_bpda_substitute(model_test.instance.model, test_loader)
model_test.eval(test_loader, eps, attack, num=ntest, norm=norm)
def seq_attack_search(f_net,
epsilon,
norm,
num_attack,
ntrials,
nbase,
search_space,
algo='tpe',
tau=4,
sample_mult=2,
timelimit=1.,
nbase_decrease_rate=1.0,
ntrials_decrease_rate=1.0,
timelimit_increase_rate=1.0,
use_vertex_removal=False,
dataset='cifar10',
sha=True,
eval=False,
logdir='log'):
"""
Main method to do the attack parameter search
"""
start_time = time.time()
device = 'cuda'
eval_result = []
attack_list = []
_, _, test_loader, _, _, _ = get_loaders(dataset, 128, 128)
data = get_data_tensor(test_loader, num=10000, offset=0, device=device)
n_samples = len(data[0])
# Network Transformation
if f_net.instance.detector is None:
is_detector = False
else:
is_detector = True
defense_property = defense_property_test(f_net.instance.model, test_loader)
defense_property.detector = is_detector
g_net = search_g_net(norm,
epsilon,
f_net,
test_loader,
defense_property,
use_vertex_removal=use_vertex_removal)
# Define Search Space
config = dict(g_net=g_net,
f_net=f_net,
defense_property=defense_property,
epsilon=epsilon,
norm=norm,
repeat=1,
timelimit=timelimit)
space = search_space.define_space(config, defense_property)
points_to_evaluate = None
# Remove non accurate samples
attackUpdater = AttackUpdater(data, f_net.instance, g_net.instance.model,
epsilon, is_detector,
defense_property.is_random)
data = attackUpdater.get_acc_img()
# Run num_attack iterations to get sequence of attacks
for attack_iter in range(num_attack):
global SEARCH_RESULT_LIST
SEARCH_RESULT_LIST = []
attack = hyperopt_sha_search(data,
space,
config,
ntrials,
nbase,
points_to_evaluate=points_to_evaluate,
algo=algo,
tau=tau,
sample_mult=sample_mult,
sha=sha,
device=device,
logdir=logdir)
if attack is None:
break # if all the attacks get time out
attack_list.append(attack)
print("The attack distribution: ")
print(collect_attack_search_statistics(SEARCH_RESULT_LIST), "\n")
# remove the samples broken by the attack
data = attackUpdater.get_robust_img([attack])
# update search parameters
ntrials = math.ceil(ntrials / ntrials_decrease_rate)
nbase = math.ceil(nbase / nbase_decrease_rate)
timelimit = timelimit * timelimit_increase_rate
# print current progress
eval_result = attackUpdateEval(attackUpdater, n_samples, start_time)
final_attack = SeqAttack(attack_list)
# If eval is True, use the attacks to perform evaluate again
if eval:
get_result_from_loader([final_attack], [g_net],
test_loader,
epsilon,
10000,
norm,
to_print=True,
eval_net=f_net)
return final_attack, g_net, eval_result
from zoo.mnist_model_pool import *
from utils.loader import get_loaders
from attack.attack_pool import *
from attack.network_transformation import *
import eagerpy as ep
device = 'cuda'
eps = 0.3
norm = ep.inf
ntest = 1000
# dataset
bounds = (0, 1)
trainN, train_loader, test_loader, input_size, input_channels, n_class = get_loaders(
'mnist', 128, 128)
# meta attack scheme
untargeted_attack = LinfProjectedGradientDescentAttack()
# attack = TargetedAttack(untargeted_attack)
attack = untargeted_attack
model_class_list = [Model1, Model2, Model3, Model4, Model5, Model6, Model7]
# model_class_list = [Model6, Model7]
for model_class in model_class_list:
model_test = model_class(load=False, device=device)
if not model_test.exists():
model_test.train(train_loader, eps, save=True)
auto_bpda_substitute(model_test.instance.model, test_loader)
model_test.eval(test_loader, eps, attack, num=ntest, norm=norm)
else:
model_test = model_class(load=True, device=device)
def density_plot(f_net,
epsilon,
norm,
ntrials,
nbase,
search_space,
algo='tpe',
tau=4,
sample_mult=2,
timelimit=1.,
use_vertex_removal=False,
dataset='cifar10',
fn='trials'):
"""
Main method to do the attack parameter search
"""
device = 'cuda'
_, _, test_loader, _, _, _ = get_loaders(dataset, 128, 128)
# model and threat model
rawdata = get_data_tensor(test_loader, num=10000, offset=0, device=device)
# 1. get network property
defense_property = defense_property_test(f_net.instance.model, test_loader)
# 2. network processing strategy
g_net = search_g_net(norm,
epsilon,
f_net,
test_loader,
defense_property,
use_vertex_removal=use_vertex_removal)
filter_attack_list = [NoAttack()]
data = get_robust_img(filter_attack_list, f_net.instance,
g_net.instance.model, rawdata, epsilon, norm)
global SEARCH_RESULT_LIST
SEARCH_RESULT_LIST = []
# 3. define search space
config = dict(g_net=g_net,
f_net=f_net,
defense_property=defense_property,
epsilon=epsilon,
norm=norm,
repeat=1,
timelimit=timelimit)
points_to_evaluate = None
algos = algo
spaces = search_space.define_space(config, defense_property)
# 4. start search in a sequential manner
get_trial_statistics(data,
spaces,
config,
ntrials,
nbase,
points_to_evaluate=points_to_evaluate,
algos=algos,
tau=tau,
sample_mult=sample_mult,
device=device,
fn=fn,
get_plot=True)
return None