def main(args):
with torch.cuda.device(1): # ??? Remove this:
labels = list(range(10))
batch_size, lr, num_workers, latent_size = int(args.batch_size), float(args.lr), int(args.num_workers), int(args.latent_size)
IM_SIZE, model_loc, num_times, iterations = int(args.IM_SIZE), Path(args.model_loc), int(args.num_times), int(args.iterations)
device = torch.device(args.device)
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
files_df['val'] = files_df['val'].sample(n=int(args.df_sample_num))
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, label=None, return_loc=True)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, label=None, return_loc=True, normalize=True)
# Train for each of the labels, here the model_loc is not an actual loc, just the base
if args.model_type == 'vae':
model_dict = {}
model_file = Path(model_loc).name
model_name = model_file.split('-')[0]
for label in labels:
model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar'
model_loc_cr = str(model_loc.parent / model_file_cr)
model_dict[label] = load_net(model_loc_cr).to(device).eval()
results = eval_gen_vae(dataloaders['val'], model_dict, device,
num_times=num_times, iterations=iterations, latent_size=latent_size)
elif args.model_type == 'cvae':
model = load_net(model_loc).to(device).eval()
results = eval_gen_cvae(dataloaders['val'], model, labels, device,
num_times=num_times, iterations=iterations, latent_size=latent_size)
SAVE_PATH = (str(model_loc).rsplit('-', 1)[:-1][0]+'_iter'+str(iterations)+
'_nt'+str(num_times)+'_nsamp'+str(args.df_sample_num)+'_results.pkl')
pickle.dump(results, open(str(SAVE_PATH), "wb"))
def main(args):
"""Evolve to find the optimal top k-pooling for each layer / block"""
print('CUDA VERSION:', torch.version.cuda)
batch_size, num_workers, IM_SIZE, epochs = int(args.batch_size), int(
args.num_workers), int(args.IM_SIZE), int(args.epochs)
device = torch.device(args.device)
NPARAMS = 4 # there are 4 blocks for topk
NPOPULATION = int(args.NPOPULATION) # population size
MAX_GENERATIONS = int(args.MAX_GENERATIONS) # number of generations
within_block_act, after_block_act = str(args.within_block_act), str(
args.after_block_act)
model_name = 'PResNetTopK-' + str(within_block_act) + '_' + str(
after_block_act)
SAVE_PATH = Path(args.SAVE_PATH)
SAVE_PATH.mkdir(parents=True, exist_ok=True)
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
group_list = [1, 1, 1, 1]
# solutions generated from N(0, 1). later transformed [0, 1] with inv cdf
es = cma.CMAEvolutionStrategy(NPARAMS * [0], 1)
history = {}
history['xbest'] = []
history['fbest'] = []
history['xfavorite'] = []
history['NPOPULATION'] = NPOPULATION
history['MAX_GENERATIONS'] = MAX_GENERATIONS
for j in tqdm(range(MAX_GENERATIONS)):
solutions = es.ask()
fitness_list = np.zeros(es.popsize)
# ??? Make generators for each generation. Does this matter? or just do it once???
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df,
batch_size,
num_workers,
size=IM_SIZE,
path_colname='path',
adv_path_colname=None,
return_loc=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df,
batch_size,
num_workers,
size=IM_SIZE,
path_colname='path',
adv_path_colname=None,
return_loc=False,
normalize=True)
# evaluate each set of learning rates, using new model each time:
for i in range(es.popsize):
# convert the normal to a topk probability:
topk_list = [norm.cdf(x) for x in solutions[i]]
# Create a model with this topk and train it:
model = get_PResNetTopK18(within_block_act=within_block_act,
after_block_act=after_block_act,
frac_list=topk_list,
group_list=group_list,
num_classes=10)
model = model.to(device)
metrics = train_net_evol(model, dataloaders, batch_size, epochs,
device)
# the fitness is the best validation accuracy *-1, because it tries to minimize
fitness_list[i] = -1 * metrics['best_val_acc']
es.tell(solutions, fitness_list)
# es.logger.add()
es.disp()
result = es.result
history['xbest'].append(result.xbest)
history['fbest'].append(result.fbest)
history['xfavorite'].append(
result.xfavorite) # this is a weird one, maybe try it out
print("fitness at generation", (j + 1), result[1])
print("local optimum discovered by solver:\n", result[0])
print("fitness score at this local optimum:", result[1])
print('es.result_pretty-------------------')
es.result_pretty()
pickle.dump(
history,
open(
str(SAVE_PATH) + '/' + str(model_name) + '_bs_' + str(batch_size) +
'_nGen_' + str(MAX_GENERATIONS) + '_nPop_' + str(NPOPULATION) +
'_ep_' + str(NPARAMS) + '_history.pkl', "wb"))
def get_adv_perf(args):
sample_num, IM_SIZE, latent_size= int(args.sample_num), int(args.IM_SIZE), int(args.latent_size)
model_loc = Path(args.model_loc)
RESULT_PATH = Path(args.model_loc).parent
device = torch.device(args.device)
distance = str(args.distance)
num_workers = 0 ## ??? what?
batch_size = 1
num_classes = 10
labels = list(range(num_classes))
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
files_df['val'] = files_df['val'].sample(n=sample_num)
# Make generators:
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=True, normalize=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=True, normalize=False)
mean = 0.1307
std = 0.3081
transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(IM_SIZE),
transforms.ToTensor(),
transforms.Normalize((mean,), (std,))])
else:
print("Incorrect Dataset")
# if args.model_type == 'vae': # then the model_loc isn't the real location
# model_dict = {}
# model_file = Path(model_loc).name
# model_name = model_file.split('-')[0]
# for label in labels:
# model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar'
# model_loc_cr = str(model_loc.parent / model_file_cr)
# model_dict[label] = load_net(model_loc_cr).to(device).eval()
# gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).eval()
# elif args.model_type == 'cvae':
# model = load_net(model_loc).to(device).eval()
# gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).eval()
# Train for each of the labels, here the model_loc is not an actual loc, just the base
if args.model_type == 'vae' or args.model_type == 'feat_vae':
model_dict = {}
model_file = Path(model_loc).name
model_name = model_file.split('-')[0]
for label in labels:
model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar'
model_loc_cr = str(model_loc.parent / model_file_cr)
model_dict[label] = load_net(model_loc_cr, args).to(device).eval()
print('args.model_type', args.model_type)
if args.model_type == 'vae':
print(f'Loading VAE')
gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval()
elif args.model_type == 'feat_vae':
print(f'Loading FEATURE VAE')
gen_model = GenerativeFeatVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval()
else:
print("Invalid model_type")
# If CVAE, load model and predict normally:
elif args.model_type == 'cvae':
print(f'Loading CVAE')
model = load_net(model_loc, args).to(device).eval()
gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).to(device).eval()
else:
print("Incorrect Model Type")
# ATTACK:
fmodel = foolbox.models.PyTorchModel(gen_model, bounds=(0, 1), preprocessing=(mean, std),
num_classes=num_classes, device=device)
# attack = get_attack(args.attack_type, fmodel, distance=distance)
num_failed = 0
results = pd.DataFrame()
for i, batch in enumerate(tqdm(dataloaders['val'])):
# Foolbox always wants numpy arrays, and we are using single batch, so this batch dim is removed.
img, label, file_loc = batch[0].to(device), batch[1].to(device), batch[2][0]
# image, label, file_loc = batch[0].numpy(), int(batch[1].numpy()), batch[2][0]
# image = load_image(file_loc) See if should use this with the foolbox preprocessign instead
# np_img = np.expand_dims(np.squeeze(img.numpy()), 0)
# np_img = np.squeeze(img.cpu().numpy())
# trans_img = transform(img.squeeze().unsqueeze(0).cpu())
# trans_img = trans_img.unsqueeze(0).to(device)
# trans_orig_pred = np.argmax(gen_model(trans_img).cpu().numpy())
np_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0)
np_label = int(label.cpu().numpy()[0])
# print('original prediction foolbox: ', np.argmax(fmodel.predictions(np_img)))
# if args.attack_type == 'boundary':
# # adv_object = attack(np_img, np_label, unpack=False, tune_batch_size=False, verbose=True, log_every_n_steps=1)
# else:
# adv_object = attack(np_img, np_label, unpack=False)
# $$$$$$$$$$$$$$$$$$
att = fa.PointwiseAttack(fmodel)
metric = foolbox.distances.L0
criterion = foolbox.criteria.Misclassification()
# Estimate gradients from scores
if not gen_model.has_grad:
GE = foolbox.gradient_estimators.CoordinateWiseGradientEstimator(0.1)
fmodel = foolbox.models.ModelWithEstimatedGradients(fmodel, GE)
# gernate Adversarial
a = foolbox.adversarial.Adversarial(fmodel, criterion, np_img, np_label, distance=metric)
att(a)
print('pred', np.argmax(fmodel.predictions(a.image)))
print('distance', a.normalized_distance)
# $$$$$$$$$$$$$$$$$$
adv_distance = adv_object.distance
np_adv_img = adv_object.image
# adv_img = torch.from_numpy(np_adv_img).float().unsqueeze(0).to(device)
# orig_pred = np.argmax(gen_model(img).cpu().numpy())
# adv_pred = np.argmax(gen_model(adv_img).cpu().numpy())
# np_trans_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0)
# trans_adv_img = attack(trans_img, np_label)
# trans_adv_pred = np.argmax(gen_model(trans_adv_img).cpu().numpy())
f_orig_pred = np.argmax(fmodel.predictions(np_img))
f_adv_pred = np.argmax(fmodel.predictions(np_adv_img))
# print(f'FOOLBOX: orig_pred: {f_orig_pred}, adv_pred: {f_adv_pred}')
# print(f'Original Label {np_label}')
if np_adv_img is None:
print('-----np_adv_img is None, Attack Failed')
num_failed +=1
results = results.append({'path': file_loc, 'true_label': np.squeeze(label.item()), 'adv_distance': adv_distance.value,
'f_orig_pred': f_orig_pred, 'f_adv_pred': f_adv_pred,
'attack_type': attack_type, 'distance': distance}, ignore_index=True)
save_loc = str(RESULT_PATH)+'/results_'+str(Path(model_loc).name)+'_ns'+str(sample_num)+'_'+str(attack_type)+'_'+str(distance)+'_adv.pkl'
print(f'saving at: {save_loc}')
with open(save_loc, 'wb') as f:
pickle.dump(results, f, pickle.HIGHEST_PROTOCOL)
print(f'Finished testing {sample_num} images. Number of failures: {num_failed}')
def main(args):
print('CUDA VERSION:', torch.version.cuda)
batch_size, num_workers = int(args.batch_size), int(args.num_workers)
IM_SIZE = int(args.IM_SIZE)
device = torch.device(args.device)
NPARAMS = int(args.epochs) # one learning rate (parameter) per epoch
NPOPULATION = int(args.NPOPULATION) # population size
MAX_GENERATIONS = int(args.MAX_GENERATIONS) # number of generations
SAVE_PATH = Path(args.SAVE_PATH)
SAVE_PATH.mkdir(parents=True, exist_ok=True)
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
# is std of 2 the right way? or too much?. Normally use +13 std for sol, but because using smaller pop use larger one.
es = cma.CMAEvolutionStrategy(NPARAMS * [-2], 2) # solutions generated from N(-2, 1), but transformed to 10^sol
history = {}
history['xbest'] = []
history['fbest'] = []
history['xfavorite'] = []
history['NPOPULATION'] = NPOPULATION
history['MAX_GENERATIONS'] = MAX_GENERATIONS
for j in tqdm(range(MAX_GENERATIONS)):
solutions = es.ask()
fitness_list = np.zeros(es.popsize)
# ??? Make generators for each generation. Does this matter? or just do it once???
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=False, normalize=True)
# evaluate each set of learning rates, using new model each time:
for i in range(es.popsize):
model, model_name = net_from_args(args, num_classes=10, IM_SIZE=IM_SIZE)
model = model.to(device)
# convert the exponenet to a learning rate:
lr_list = np.power(10, solutions[i]).tolist()
# Train it for the given lr list:
metrics = train_net(model, dataloaders, lr_list, batch_size, device)
# the fitness is the best validation accuracy *-1, because it tries to minimize
fitness_list[i] = -1 * metrics['best_val_acc']
es.tell(solutions, fitness_list)
# es.logger.add()
es.disp()
result = es.result
history['xbest'].append(result.xbest)
history['fbest'].append(result.fbest)
history['xfavorite'].append(result.xfavorite) # this is a weird one, maybe try it out
print("fitness at generation", (j+1), result[1])
print("local optimum discovered by solver:\n", result[0])
print("fitness score at this local optimum:", result[1])
print('es.result_pretty-------------------')
es.result_pretty()
pickle.dump(history, open(str(SAVE_PATH)+'/'+str(model_name)+'_bs_'+str(batch_size)+'_nGen_'+str(MAX_GENERATIONS)+'_nPop_'+str(NPOPULATION)+'_ep_'+str(NPARAMS)+'_history.pkl', "wb"))
def main(args):
with torch.cuda.device(1): # ??? Remove this:
if args.layer_sizes:
args.layer_sizes = [int(i) for i in args.layer_sizes]
epochs, batch_size, lr, num_workers = int(args.epochs), int(args.batch_size), float(args.lr), int(args.num_workers)
num_labels, IM_SIZE= int(args.num_labels), int(args.IM_SIZE)
device = torch.device(args.device)
SAVE_PATH = Path(args.SAVE_PATH)
SAVE_PATH.mkdir(parents=True, exist_ok=True)
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
# Train for each of the labels:
for label in range(num_labels):
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, label=label, return_loc=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, label=label, return_loc=False)
# get the network
model, model_name = vae_from_args(args)
model = model.to(device)
print('next(model.parameters()).device', next(model.parameters()).device)
print(f'--------- Training: {model_name} ---------')
# get training parameters and train:
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, step_size=int(epochs/4), gamma=0.2) # close enough
metrics = {}
metrics['train_losses'] = []
metrics['val_losses'] = []
best_val_loss = 100000000
criterion = model.loss
for epoch in range(epochs):
# train for one epoch
train_losses = train_epoch_auto(epoch, lr, dataloaders['train'], model, optimizer, criterion, device)
metrics['train_losses'].append(train_losses)
# evaluate on validation set
val_losses = validate_epoch_auto(dataloaders['val'], model, criterion, device)
metrics['val_losses'].append(val_losses)
scheduler.step()
# remember best validation accuracy and save checkpoint
is_best = val_losses < best_val_loss
best_val_loss = min(val_losses, best_val_loss)
save_checkpoint({
'model_name': model_name,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict(),
'epoch': epoch + 1,
'best_val_loss': best_val_loss,
'metrics': metrics,
}, is_best, model_name+'_label_'+str(label), SAVE_PATH)
RESULT_PATH = str(SAVE_PATH)+'/'+str(model_name)+'_label_'+str(label)+'_metrics.pkl'
print('Saving results at:', RESULT_PATH)
pickle.dump(metrics, open(RESULT_PATH, "wb"))
def main(args):
with torch.cuda.device(1):
epochs, batch_size, lr, num_workers = int(args.epochs), int(args.batch_size), float(args.lr), int(args.num_workers)
IM_SIZE = int(args.IM_SIZE)
device = torch.device(args.device)
SAVE_PATH = Path(args.SAVE_PATH)
SAVE_PATH.mkdir(parents=True, exist_ok=True)
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
# Make generators:
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=False, normalize=True)
# get the network
model, model_name = net_from_args(args, num_classes=10, IM_SIZE=IM_SIZE)
model = model.to(device)
print(f'--------- Training: {model_name} ---------')
# get training parameters and train:
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=5e-4)
scheduler = lr_scheduler.StepLR(optimizer, step_size=int(epochs/3), gamma=0.2) # close enough
metrics = {}
metrics['train_top1_acc'] = []
metrics['train_losses'] = []
metrics['val_top1_acc'] = []
metrics['val_losses'] = []
best_val_acc = 0
for epoch in tqdm(range(epochs)):
# train for one epoch
train_top1_acc, train_losses = train_epoch(dataloaders['train'], model, criterion, optimizer, epoch, device)
metrics['train_top1_acc'].append(train_top1_acc)
metrics['train_losses'].append(train_losses)
# evaluate on validation set
val_top1_acc, val_losses = validate_epoch(dataloaders['val'], model, device, criterion=None)
metrics['val_top1_acc'].append(val_top1_acc)
metrics['val_losses'].append(val_losses)
scheduler.step()
# remember best validation accuracy and save checkpoint
is_best = val_top1_acc > best_val_acc
best_val_acc = max(val_top1_acc, best_val_acc)
save_checkpoint({
'model_name': model_name,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict(),
'epoch': epoch + 1,
'best_val_acc': best_val_acc,
'metrics': metrics,
}, is_best, model_name, SAVE_PATH)
pickle.dump(metrics, open(str(SAVE_PATH)+'/'+str(model_name)+'_metrics.pkl', "wb"))
def main(args):
batch_size, lr, num_workers, latent_size = int(args.batch_size), float(
args.lr), int(args.num_workers), int(args.latent_size)
IM_SIZE, model_loc, num_times, iterations = int(args.IM_SIZE), Path(
args.model_loc), int(args.num_times), int(args.iterations)
device = torch.device(args.device)
deter = args.deter
num_workers = 2 ## do 0 if doesnt work
num_classes = 10
labels = list(range(num_classes))
all_results = pd.DataFrame()
# Get file sample and make generators:
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
files_df['val'] = files_df['val'].sample(n=int(args.df_sample_num))
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df,
batch_size,
num_workers,
size=IM_SIZE,
path_colname='path',
adv_path_colname=None,
return_loc=True)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df,
batch_size,
num_workers,
size=IM_SIZE,
path_colname='path',
adv_path_colname=None,
return_loc=True,
normalize=True)
# Train for each of the labels, here the model_loc is not an actual loc, just the base
if args.model_type == 'vae' or args.model_type == 'feat_vae':
print(f'Loading VAE with Deterministic forward pass: {deter}')
model_dict = {}
model_file = Path(model_loc).name
model_name = model_file.split('-')[0]
for label in labels:
model_file_cr = model_file + '_label_' + str(
label) + '_model_best.pth.tar'
model_loc_cr = str(model_loc.parent / model_file_cr)
model_dict[label] = load_net(model_loc_cr, args).to(device).eval()
if args.model_type == 'vae':
gen_model = GenerativeVAE(model_dict=model_dict,
labels=labels,
latent_size=latent_size,
device=device)
elif args.model_type == 'feat_vae':
gen_model = GenerativeFeatVAE(model_dict=model_dict,
labels=labels,
latent_size=latent_size,
device=device)
else:
print("Invalid model_type")
# If CVAE, load model and predict normally:
elif args.model_type == 'cvae':
print(f'Loading CVAE with Deterministic forward pass: {deter}')
model = load_net(model_loc, args).to(device).eval()
gen_model = GenerativeCVAE(model=model,
labels=labels,
latent_size=latent_size,
device=device).eval()
with torch.no_grad():
for i, (tensor_img, label,
path) in enumerate(tqdm(dataloaders['val'])):
tensor_img, label, file_loc = tensor_img.to(device), label.to(
device), path[0]
preds, results = gen_model(tensor_img,
iterations=iterations,
num_times=num_times,
info=True,
deterministic=deter)
pred = np.argmax(preds.cpu().numpy())
img_results = {} # convert to dict so can add to pandas
for ind in labels:
img_results[ind] = results[ind]
img_results['path'] = path
img_results['true_label'] = int(label.cpu().numpy())
img_results['path'] = path
img_results['predicted_label'] = int(pred)
all_results = all_results.append(img_results, ignore_index=True)
SAVE_PATH = (str(model_loc).rsplit('-', 1)[:-1][0] + '_iter' +
str(iterations) + '_nt' + str(num_times) + '_nsamp' +
str(args.df_sample_num))
if deter:
SAVE_PATH = SAVE_PATH + '_deter'
SAVE_PATH = SAVE_PATH + '_results.pkl'
print(f'saving at: {SAVE_PATH}')
pickle.dump(all_results, open(str(SAVE_PATH), "wb"))