def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print(
'-------------------------------------------------------------------')
print('setup_direction')
print(
'-------------------------------------------------------------------')
# Setup env for preventing lock on h5py file for newer h5py versions
# added due to the commit https://github.com/tomgoldstein/loss-landscape/pull/28/files
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print("%s is already set up" % dir_file)
return
f.close()
# Create the plotting directions
# 下面介绍中, xdir 即 xdirection, ydir 即 ydirection
# 第一层选择: 两个模型, or 一个模型, or 三个模型
# 第二层选择: 画1d 还是 2d
# 两个模型 + 只画 x: 只有 xdir= model2 - model1
# (少见) 两个模型 + 画x,y: xdir = model2 - model 1, 但另一个方向ydirection 是随机
# (少见) 一个模型 + 只画 x: 只有 xdir = random.
# 一个模型 + 画 x, y: xdir 和 ydir 都是随机
# 三个模型 + 画 x, y: xdir = model2 - model1, ydir = model3 - model 1.
f = h5py.File(dir_file, 'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2: # If extra model is provided, then only check xdirection
net2 = model_loader.load(args.dataset, args.model,
args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else: # If no extra file, then use a random direction. Use filter normalization by default
xdirection = create_random_direction(net, args.dir_type,
args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y: # If we want to draw 2d plots:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model,
args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type,
args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print(
'-------------------------------------------------------------------')
print('setup_direction')
print(
'-------------------------------------------------------------------')
# Setup env for preventing lock on h5py file for newer h5py versions
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print("%s is already setted up" % dir_file)
return
f.close()
# Create the plotting directions
f = h5py.File(dir_file, 'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2:
net2 = model_loader.load(args.dataset, args.model,
args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else:
xdirection = create_random_direction(net, args.dir_type,
args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model,
args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type,
args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created: %s" % dir_file)
def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(
'The projection file exists! No projection is performed unless %s is deleted'
% proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def convert_matlab_pca_data(args, direction_matlab_name,
direction_python_name):
# class ARGS:
# dataset='cifar10'
# model='resnet56'
# model_folder='folders for models to be projected'
# dir_type='weights'
# ignore='biasbn'
# prefix='model_'
# suffix='.t7'
# start_epoch=0
# max_epoch=500
# save_epoch=1
# args = ARGS()
# args.model_folder = model_folder
# args.model = model
last_model_file = args.model_folder + '/' + args.prefix + str(
args.max_epoch) + args.suffix
net = model_loader.load(args.dataset, args.model, last_model_file)
w = net_plotter.get_weights(net)
# read in matlab pca results
f = h5py.File(direction_matlab_name, 'r')
fpy = h5py.File(direction_python_name, 'w')
fpy['explained_variance_ratio_'] = np.array(f['explained_variance_ratio_'])
fpy['explained_variance_'] = np.array(f['explained_variance_'])
pc1 = np.array(f['directionx'])
pc2 = np.array(f['directiony'])
f.close()
# convert vectorized directions to the same shape as models to save in h5 file.
# import pdb; pdb.set_trace()
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
# import pdb; pdb.set_trace()
h5_util.write_list(fpy, 'xdirection', xdirection)
h5_util.write_list(fpy, 'ydirection', ydirection)
fpy.close()
print('PCA directions saved in: %s' % direction_python_name)
def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print('-------------------------------------------------------------------')
print('setup_direction')
print('-------------------------------------------------------------------')
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print ("%s is already setted up" % dir_file)
return
f.close()
# Create the plotting directions
f = h5py.File(dir_file,'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2:
net2 = model_loader.load(args.dataset, args.model, args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else:
xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model, args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print ("direction file created: %s" % dir_file)
def project_trajectory(dir_file, w, s, dataset, model_name, model_files,
dir_type='weights', proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print('The projection file exists! No projection is performed unless %s is deleted' % proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print ("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
args.ymin, args.ymax, args.ynum = (None, None, None)
if args.y:
args.ymin, args.ymax, args.ynum = [
float(a) for a in args.y.split(':')
]
assert args.ymin and args.ymax and args.ynum, \
'You specified some arguments for the y axis, but not all'
except:
raise Exception(
'Improper format for x- or y-coordinates. Try something like -1:1:51'
)
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
net = model_loader.load(args.dataset, args.model, args.model_file)
w = net_plotter.get_weights(net) # initial parameters
s = copy.deepcopy(
net.state_dict()) # deepcopy since state_dict are references
if args.ngpu > 1:
# data parallel with multiple GPUs on a single node
net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
surf_file = name_surface_file(args, dir_file)
args.ymin, args.ymax, args.ynum = (None, None, None)
if args.y:
args.ymin, args.ymax, args.ynum = [
float(a) for a in args.y.split(':')
]
assert args.ymin and args.ymax and args.ynum, \
'You specified some arguments for the y axis, but not all'
except:
raise Exception(
'Improper format for x- or y-coordinates. Try something like -1:1:51'
)
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
net = model_loader.load(args.dataset, args.model, args.model_file,
args.parallel, args.auxiliary)
w = net_plotter.get_weights(net) # initial parameters
s = copy.deepcopy(
net.state_dict()) # deepcopy since state_dict are references
if args.ngpu > 1:
# data parallel with multiple GPUs on a single node
net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
surf_file = name_surface_file(args, dir_file)
def main():
#--------------------------------------------------
# Parse input arguments
#--------------------------------------------------
parser = argparse.ArgumentParser(
description=
'Generation and evaluation of VGG-ANN crafted adversarial attack on the CIFAR10 or CIFAR100 dataset',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--seed', default=2, type=int, help='Random seed')
parser.add_argument('--dataset',
default='CIFAR10',
type=str,
help='{CIFAR10, CIFAR100} dataset type')
parser.add_argument('--arch',
default='VGG5',
type=str,
help='{VGG5, VGG11} network architecture')
parser.add_argument('--log', default=None, type=str, help='Log file name')
#-------------------------------------------------------
# dataset parameters
#--------------------------------------------------------
parser.add_argument(
'--mean',
default='0.5,0.5,0.5',
type=str,
help='mean of each channel of image in the format mean1,mean2,mean3')
parser.add_argument(
'--std',
default='0.5,0.5,0.5',
type=str,
help='std of each channel of image in the format std1,std2,std3')
parser.add_argument('--batch_size',
default=64,
type=int,
help='Batch size')
#------------------------------------------------------
# SNN parameters
#-------------------------------------------------------
parser.add_argument('--timesteps',
default=100,
type=int,
help='Total number of time steps')
parser.add_argument('--leak_mem',
default=0.99,
type=float,
help='Leak in membrane potential for LIF neuron')
#-------------------------------------------------------
# Adversarial attack parameters
#---------------------------------------------------------
parser.add_argument('--epsilon',
default=8,
type=int,
help='Attack intensity. Possible values: 2,4,6,8,16')
parser.add_argument(
'--eps_iter',
default=2,
type=int,
help=
'Attack intensity per iteration for PGD attack. Possible values:1,2')
parser.add_argument('--pgd_steps',
default=7,
type=int,
help='#steps in pgd. Possible values: 7, 10, 20, 40')
parser.add_argument(
'--rand_init',
default=1,
type=int,
help=
'randomly perturb the original image in l-inf ball of epsilon for pgd. Possible values: 1, 0'
)
parser.add_argument(
'--source',
default='ann',
type=str,
help='source network. Possible values: ann|snnconv|snnbp')
parser.add_argument(
'--target',
default='ann',
type=str,
help='target network. Possible values: ann|snnconv|snnbp')
parser.add_argument('--attack',
default='fgsm',
type=str,
help='attack category. Possible values: fgsm|pgd')
parser.add_argument(
'--type',
default='wb',
type=str,
help='Whitebox or Blackbox attack. possible values: wb|bb')
parser.add_argument('--targeted',
default='False',
type=str,
help='True|False')
global args
args = parser.parse_args()
#--------------------------------------------------
# Initialize seed
#--------------------------------------------------
seed = args.seed
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
#-----------------------------------------------------
# dataset and architecture
#-----------------------------------------------------
dataset = args.dataset
architecture = args.arch
#-----------------------------------------------------
# batch size
#----------------------------------------------------
batch_size = args.batch_size
#-----------------------------------------------------
# Adversarial attack parameters
#-----------------------------------------------------
model_source_name = args.source
model_target_name = args.target
attack = args.attack
targeted = args.targeted
epsilon = args.epsilon
eps_iter = args.eps_iter
num_iter = args.pgd_steps
rand_init = args.rand_init
attack_type = args.type
clip_min = 0.0
clip_max = 1.0
#---------------------------------------------------
# log file
#----------------------------------------------------
logfile_name = dataset.lower() + architecture.lower(
) + '_' + model_source_name + '_crafted_on_' + model_target_name + '_' + attack_type + '_' + attack + '.log'
f = open(logfile_name, 'a', buffering=1)
#--------------------------------------------------
# Load the CIFAR10 dataset
#--------------------------------------------------
data_root = expanduser("~")
# These are the mean and std values used to train the models
args.mean1, args.mean2, args.mean3 = [
float(a) for a in args.mean.split(',')
]
args.std1, args.std2, args.std3 = [float(a) for a in args.std.split(',')]
mean = [args.mean1, args.mean2, args.mean3]
std = [args.std1, args.std2, args.std3]
transform_test = transforms.Compose([transforms.ToTensor()])
if dataset == 'CIFAR10':
testset = datasets.CIFAR10(root=data_root,
train=False,
download=True,
transform=transform_test)
testloader = DataLoader(testset,
batch_size=batch_size,
shuffle=False,
drop_last=True)
#--------------------------------------------------
# Instantiate the Source & Target model
#--------------------------------------------------
model_dir = os.getcwd()
# map between model name and model_file directory
model_source_dir = {
'ann_wb': model_dir + '/ann_checkpoint1.pt',
'ann_bb': model_dir + '/ann_checkpoint2.pt',
'snnconv_wb': model_dir + '/ann_checkpoint1.pt',
'snnconv_bb': model_dir + '/ann_checkpoint2.pt',
'snnbp_wb': model_dir + '/snnbp_checkpoint1.pt',
'snnbp_bb': model_dir + '/snnbp_checkpoint2.pt'
}
# The threshold values of SNN models. Two sets of thresholds corresponding to two separately initialized models
snn_thresholds = [[20.76, 2.69, 2.33, 0.28, 1.14],
[20.86, 2.74, 2.21, 0.38, 0.99]] # VGG5
# wb: whitebox, bb: blackbox
if attack_type == 'wb':
threshold_id = 0
elif attack_type == 'bb':
threshold_id = 1
model_source = model_loader.load(
model_source_name, batch_size,
model_source_dir[model_source_name + '_' + attack_type],
snn_thresholds[threshold_id][0:])
model_target = model_loader.load(
model_target_name, batch_size,
model_source_dir[model_target_name + '_wb'], snn_thresholds[0][0:])
# Used during analyzing the variation of number of timesteps and leak factor
# if model_target.module.type == 'SNN':
# model_target.module.timesteps_init(args.timesteps)
# if model_target_name == 'snnbp':
# model_target.module.leak_init(args.leak_mem)
#--------------------------------------------------
# Initialize the loss function
#--------------------------------------------------
loss_func = nn.CrossEntropyLoss().cuda()
# Print the simulation parameters
print('Batch size : {}'.format(batch_size))
print('{}-crafted attack on {}:{}'.format(model_source_name,
model_target_name, attack_type))
print('Attack category: {}'.format(attack))
print('epsilon:{}|targeted:{}'.format(epsilon, targeted))
if attack == 'pgd':
print('epsilon_iter:{}|num_iter:{}'.format(eps_iter, num_iter))
print('*********************************************************')
#---------------------------------------------------------------------
# Adversarial attack generation and evaluation
#--------------------------------------------------------------------
count_clean = 0 # keeps track of the correct predictions for clean input
count_adv = 0 # keeps track of the correct predictions for the adversarial input
# Instantiate the attack model
if attack == 'fgsm':
attack_model = FastGradientSign(epsilon * 1.0 / 255.0, clip_min,
clip_max, targeted)
elif attack == 'pgd':
attack_model = ProjectedGradientDescent(num_iter,
epsilon * 1.0 / 255.0,
eps_iter * 1.0 / 255.0,
clip_min, clip_max, targeted,
rand_init, seed)
for batch_id, (images, labels) in enumerate(testloader):
images, labels = images.cuda(), labels.cuda()
# Calculate the clean test accuracy of the target model
acc, loss = eval_minibatch(model_target, images, labels, mean, std,
loss_func)
count_clean += acc
# create target label in case of 'targeted' attack
if targeted == 'False':
target_labels = labels
else:
target_labels = random_targets(10, labels.cpu(), seed)
target_labels = target_labels.cuda()
# Generate adversarial example
images_adv = attack_model.generate(images, labels, model_source,
loss_func, mean, std)
# Calculate the adv test accuracy of the target model
acc, loss = eval_minibatch(model_target, images_adv, target_labels,
mean, std, loss_func)
count_adv += acc
# Print the clean and adv count
if (batch_id != 0) and (batch_id % 2 == 0):
print('Minibatch : {}'.format(batch_id))
print('clean/adv: {}/{}\n'.format(int(count_clean),
int(count_adv)))
# Adversarial loss = precision_clean - precision_adv
precision_adv = float(count_adv) * 100 / len(testloader.dataset)
precision_clean = float(count_clean) * 100 / len(testloader.dataset)
print('Adversarial Loss: {:.3f}%\n'.format(precision_clean -
precision_adv))
if attack == 'pgd':
f.write(
"\t {}:{}:{}|targeted:{}|eps_iter:{}, num_iter:{}--> adv loss = {:.3f}%\n"
.format(model_source_name, model_target_name, attack_type,
targeted, eps_iter, num_iter,
precision_clean - precision_adv))
elif attack == 'fgsm':
f.write("\t {}:{}:{}|targeted:{}|epsilon:{}--> adv loss = {:.3f}%\n".
format(model_source_name, model_target_name, attack_type,
targeted, epsilon, precision_clean - precision_adv))
f.close()
def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos',
data_parallel=False):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
save_epoch: the checkpoint frequency
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(proj_file + ' exits! No projection is performed.')
return proj_file
# read directions and convert them to vectors
f = h5py.File(dir_file, 'r')
directions = net_plotter.load_directions(f)
dx = list_to_vec(directions[0])
dy = list_to_vec(directions[1])
f.close()
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file,
data_parallel)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = weights_to_vec(d)
x, y = project_2D(d, dx, dy, proj_method)
print(model_file, x, y)
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
from flask import Flask, request
from flask import jsonify
from model_loader import load
model_name = "model.bin"
model = load(model_name)
app = Flask(__name__)
@app.route('/vec/<term>')
def vec(term):
return jsonify(model[term].tolist())
@app.route('/vecs', methods=['POST'])
def vecs():
if request.method == 'POST':
terms = request.get_json()
w2vecs = [model[term].tolist() for term in terms]
return jsonify(w2vecs)
return 'Only post request allowed'
# in case of multiple GPUs per node, set the GPU to use for each rank
if args.cuda:
if not torch.cuda.is_available():
raise Exception(
'User selected cuda option, but cuda is not available on this machine'
)
gpu_count = torch.cuda.device_count()
print('Use GPU %d of %d GPUs on %s' %
(torch.cuda.current_device(), gpu_count, socket.gethostname()))
for epoch in range(0, args.max_epoch, 5):
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
net = model_loader.load(
args.dataset, args.model,
args.model_folder + '/model_' + str(epoch) + '.t7')
if args.ngpu > 1:
# data parallel with multiple GPUs on a single node
net = nn.DataParallel(net,
device_ids=range(torch.cuda.device_count()))
#--------------------------------------------------------------------------
# Setup dataloader
#--------------------------------------------------------------------------
# download CIFAR10 if it does not exit
if args.dataset == 'cifar10':
torchvision.datasets.CIFAR10(root=args.dataset + '/data',
train=True,
download=True)
args.ymin, args.ymax, args.ynum = [
float(a) for a in args.y.split(':')
]
assert args.ymin and args.ymax and args.ynum, \
'You specified some arguments for the y axis, but not all'
except:
raise Exception(
'Improper format for x- or y-coordinates. Try something like -1:1:51'
)
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
net = model_loader.load(args.dataset,
args.model,
args.model_file,
data_parallel=args.data_parallel,
num_blocks=args.num_blocks)
w = net_plotter.get_weights(net) # initial parameters
s = copy.deepcopy(
net.state_dict()) # deepcopy since state_dict are references
if args.ngpu > 1:
# data parallel with multiple GPUs on a single node
net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
s1_BCH = pd.read_csv('BCHs1_no_ask.csv').values
s2_BCH = pd.read_csv('BCHs2_no_ask.csv').values
s3_BCH = pd.read_csv('BCHs3_no_ask.csv').values
s1_ETH = pd.read_csv('ETHs1_no_ask.csv').values
s2_ETH = pd.read_csv('ETHs2_no_ask.csv').values
s3_ETH = pd.read_csv('ETHs3_no_ask.csv').values
s1_LTC = pd.read_csv('LTCs1_no_ask.csv').values
s2_LTC = pd.read_csv('LTCs2_no_ask.csv').values
s3_LTC = pd.read_csv('LTCs3_no_ask.csv').values
# pre-load model to speed up
gru_models = {}
for asset in assets:
gru_models[asset] = model_loader.load(asset)
upper_bound = [516.77, 6546.40, 224.39, 58.64]
lower_bound = [486.58, 6481.08, 218.36, 56.93]
def predict_dpi(x, s):
"""Predict the average price change Δp_i, 1 <= i <= 3.
Args:
x: A numpy array of floats representing previous prices.
s: A 2-dimensional numpy array generated by choose_effective_centers().
Returns:
A big float representing average price change Δp_i.
"""
def setup_PCA_directions(args, model_files, w, s):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print (model_file)
net2 = model_loader.load(args.dataset, args.model, model_file)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print ("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_))
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print ('PCA directions saved in: %s' % dir_name)
return dir_name
# if int(epoch) == 150 or int(epoch) == 225 or int(epoch) == 275:
# lr *= args.lr_decay
# for param_group in optimizer.param_groups:
# param_group['lr'] *= args._lr_decay
f.close()
sio.savemat('trained_nets/' + save_folder + '/' + args.model + '_gradient_noise.mat',
mdict={'training_history': training_history,'testing_history': testing_history,'train_noise_norm': noise_norm_history_TRAIN,'weight_grad_history': weight_grad_history},
)
#--------------------------------------------------------------------------
# Load weights and save them in a mat file (temporal unit: epoch)
#--------------------------------------------------------------------------
all_weights = []
for i in range(0,args.epochs+1,args.save_epoch):
model_file = 'trained_nets/' + save_folder + '/' + 'model_' + str(i) + '.t7'
net = ml.load('cifar10', args.model, model_file)
w = net_plotter.get_weights(net) # initial parameters
#s = copy.deepcopy(net.state_dict()) # deepcopy since state_dict are references
#import pdb; pdb.set_trace()
for j in range(len(w)):
w[j] = w[j].cpu().numpy()
all_weights.append(w)
sio.savemat('trained_nets/' + save_folder + '/' + args.model + 'all_weights.mat',
mdict={'weight': all_weights},
)
f = open('trained_nets/' + save_folder + '/log.out', 'a')
trainloader, testloader = dataloader.get_data_loaders(args)
if args.label_corrupt_prob and not args.resume_model:
torch.save(trainloader,
'trained_nets/' + save_folder + '/trainloader.dat')
torch.save(testloader,
'trained_nets/' + save_folder + '/testloader.dat')
# Model
if args.resume_model:
# Load checkpoint.
print('==> Resuming from checkpoint..')
checkpoint = torch.load(args.resume_model)
net = model_loader.load(args.model)
net.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch'] + 1
else:
net = model_loader.load(args.model)
print(net)
init_params(net)
if args.ngpu > 1:
net = torch.nn.DataParallel(net)
criterion = nn.CrossEntropyLoss()
if args.loss_name == 'mse':
criterion = nn.MSELoss()
if use_cuda:
# Check plotting resolution
#--------------------------------------------------------------------------
try:
args.xmin, args.xmax, args.xnum = [float(a) for a in args.x.split(':')]
args.ymin, args.ymax, args.ynum = (None, None, None)
if args.y:
args.ymin, args.ymax, args.ynum = [float(a) for a in args.y.split(':')]
assert args.ymin and args.ymax and args.ynum, \
'You specified some arguments for the y axis, but not all'
except:
raise Exception('Improper format for x- or y-coordinates. Try something like -1:1:51')
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
net = model_loader.load(args.dataset, args.model, args.model_file)
w = net_plotter.get_weights(net) # initial parameters
s = copy.deepcopy(net.state_dict()) # deepcopy since state_dict are references
if args.ngpu > 1:
# data parallel with multiple GPUs on a single node
net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
surf_file = name_surface_file(args, dir_file)
if rank == 0:
def setup_PCA_directions(args, model_files):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print(model_file)
net2 = model_loader.load(args.dataset, args.model, model_file,
args.data_parallel)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = weights_to_vec(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: " + str(cal_angle(pc1, pc2)))
print(pca.explained_variance_ratio_)
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = vec_to_weights(pc1, w)
ydirection = vec_to_weights(pc2, w)
elif args.dir_type == 'states':
xdirection = vec_to_states(pc1, s)
ydirection = vec_to_states(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print('PCA directions saved in ' + dir_name)
return dir_name
states (include BN.running_mean/var)""")
parser.add_argument('--ignore', default='', help='ignore bias and BN paras: biasbn (no bias or bn)')
parser.add_argument('--prefix', default='model_', help='prefix for the checkpint model')
parser.add_argument('--suffix', default='.pth', help='prefix for the checkpint model')
parser.add_argument('--start_epoch', default=0, type=int, help='min index of epochs')
parser.add_argument('--max_epoch', default=300, type=int, help='max number of epochs')
parser.add_argument('--save_epoch', default=1, type=int, help='save models every few epochs')
parser.add_argument('--dir_file', default='', help='load the direction file for projection')
parser.add_argument('--data_parallel', action='store_true', default=False,
help='the models are saved in data_parallel format')
args = parser.parse_args()
# load the final model
last_model_file = args.model_folder + '/' + args.prefix + str(args.max_epoch) + args.suffix
net = model_loader.load(args.dataset, args.model, last_model_file, args.data_parallel)
w = net_plotter.get_weights(net)
s = net.state_dict()
# collect models to be projected
model_files = []
for epoch in range(args.start_epoch, args.max_epoch + args.save_epoch, args.save_epoch):
model_file = args.model_folder + '/' + args.prefix + str(epoch) + args.suffix
assert os.path.exist(model_file), 'model %s does not exist' % model_file
model_files.append(model_file)
# load or create projection directions
if args.dir_file:
dir_file = args.dir_file
else:
dir_file = setup_PCA_directions(args, model_files)
