def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print(
'-------------------------------------------------------------------')
print('setup_direction')
print(
'-------------------------------------------------------------------')
# Setup env for preventing lock on h5py file for newer h5py versions
# added due to the commit https://github.com/tomgoldstein/loss-landscape/pull/28/files
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print("%s is already set up" % dir_file)
return
f.close()
# Create the plotting directions
# 下面介绍中, xdir 即 xdirection, ydir 即 ydirection
# 第一层选择: 两个模型, or 一个模型, or 三个模型
# 第二层选择: 画1d 还是 2d
# 两个模型 + 只画 x: 只有 xdir= model2 - model1
# (少见) 两个模型 + 画x,y: xdir = model2 - model 1, 但另一个方向ydirection 是随机
# (少见) 一个模型 + 只画 x: 只有 xdir = random.
# 一个模型 + 画 x, y: xdir 和 ydir 都是随机
# 三个模型 + 画 x, y: xdir = model2 - model1, ydir = model3 - model 1.
f = h5py.File(dir_file, 'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2: # If extra model is provided, then only check xdirection
net2 = model_loader.load(args.dataset, args.model,
args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else: # If no extra file, then use a random direction. Use filter normalization by default
xdirection = create_random_direction(net, args.dir_type,
args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y: # If we want to draw 2d plots:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model,
args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type,
args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created: %s" % dir_file)
def convert_matlab_pca_data(args, direction_matlab_name,
direction_python_name):
# class ARGS:
# dataset='cifar10'
# model='resnet56'
# model_folder='folders for models to be projected'
# dir_type='weights'
# ignore='biasbn'
# prefix='model_'
# suffix='.t7'
# start_epoch=0
# max_epoch=500
# save_epoch=1
# args = ARGS()
# args.model_folder = model_folder
# args.model = model
last_model_file = args.model_folder + '/' + args.prefix + str(
args.max_epoch) + args.suffix
net = model_loader.load(args.dataset, args.model, last_model_file)
w = net_plotter.get_weights(net)
# read in matlab pca results
f = h5py.File(direction_matlab_name, 'r')
fpy = h5py.File(direction_python_name, 'w')
fpy['explained_variance_ratio_'] = np.array(f['explained_variance_ratio_'])
fpy['explained_variance_'] = np.array(f['explained_variance_'])
pc1 = np.array(f['directionx'])
pc2 = np.array(f['directiony'])
f.close()
# convert vectorized directions to the same shape as models to save in h5 file.
# import pdb; pdb.set_trace()
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
# import pdb; pdb.set_trace()
h5_util.write_list(fpy, 'xdirection', xdirection)
h5_util.write_list(fpy, 'ydirection', ydirection)
fpy.close()
print('PCA directions saved in: %s' % direction_python_name)
def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print(
'-------------------------------------------------------------------')
print('setup_direction')
print(
'-------------------------------------------------------------------')
# Setup env for preventing lock on h5py file for newer h5py versions
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print("%s is already setted up" % dir_file)
return
f.close()
# Create the plotting directions
f = h5py.File(dir_file, 'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2:
net2 = model_loader.load(args.dataset, args.model,
args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else:
xdirection = create_random_direction(net, args.dir_type,
args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model,
args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type,
args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, net):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print('-------------------------------------------------------------------')
print('setup_direction')
print('-------------------------------------------------------------------')
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print ("%s is already setted up" % dir_file)
return
f.close()
# Create the plotting directions
f = h5py.File(dir_file,'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
if args.model_file2:
net2 = model_loader.load(args.dataset, args.model, args.model_file2)
xdirection = create_target_direction(net, net2, args.dir_type)
else:
xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y:
if args.same_dir:
ydirection = xdirection
elif args.model_file3:
net3 = model_loader.load(args.dataset, args.model, args.model_file3)
ydirection = create_target_direction(net, net3, args.dir_type)
else:
ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print ("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, model):
"""
Setup the h5 file to store the directions.
- xdirection, ydirection: The pertubation direction added to the mdoel.
The direction is a list of tensors.
"""
print(
'-------------------------------------------------------------------')
print('setup_direction')
print(
'-------------------------------------------------------------------')
# Skip if the direction file already exists
if exists(dir_file):
f = h5py.File(dir_file, 'r')
if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys():
f.close()
print("%s is already setted up" % dir_file)
return
f.close()
# Create the plotting directions
f = h5py.File(dir_file, 'w') # create file, fail if exists
if not args.dir_file:
print("Setting up the plotting directions...")
xdirection = create_random_direction(model, args.dir_type,
args.xignore, args.xnorm)
h5_util.write_list(f, 'xdirection', xdirection)
if args.y:
if args.same_dir:
ydirection = xdirection
else:
ydirection = create_random_direction(model, args.dir_type,
args.yignore, args.ynorm)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created: %s" % dir_file)
def setup_PCA_directions(args, model_files, w, s):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print (model_file)
net2 = model_loader.load(args.dataset, args.model, model_file)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print ("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_))
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print ('PCA directions saved in: %s' % dir_name)
return dir_name
def setup_PCA_directions(args, model_files, w, s):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print(model_file)
net2 = model_loader.load(args.dataset, args.model, model_file)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" %
str(pca.explained_variance_ratio_))
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print('PCA directions saved in: %s' % dir_name)
return dir_name
def setup_othermodels_PCA_directions(args, ignore_embedding, model_files, w):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
# Name the .h5 file that stores the PCA directions.
folder_name = 'fairseq_master' + '/' + args.save_dir + '/PCA_'
folder_name += 'lr=' + str(args.lr[0])
folder_name += '_optimier=' + str(args.optimizer)
folder_name += '_ignore_embedding=' + str(ignore_embedding)
if args.ignore:
folder_name += '_ignoreBN'
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print(model_file)
state = torch.load(
model_file,
map_location=lambda s, l: torch.serialization.
default_restore_location(s, 'cpu'),
)
args2 = state['args']
args2.data = 'fairseq_master/' + args2.data
# task = tasks.setup_task(args2)
# model2 = task.build_model(args2)
# s2 = model2.state_dict()
s2 = state['model']
w2 = []
if ignore_embedding:
for key in s2:
if 'version' in key or '_float_tensor' in key or 'embed' in key:
s2[key].fill_(0)
w2.append(s2[key])
else:
for key in s2:
if 'version' in key or '_float_tensor' in key:
s2[key].fill_(0)
w2.append(s2[key])
d = net_plotter.get_diff_weights(w, w2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" %
str(pca.explained_variance_ratio_))
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print('PCA directions saved in: %s' % dir_name)
return dir_name
if __name__ == "__main__":
gpus = tf.config.experimental.list_physical_devices(
'GPU') #must limit gpu memory growth
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
model_path = "D:/Rain/text/Python/MA_IIIT/models/vgg9/vgg9_sgd_lr=0.1_bs=128_wd=0.0_epochs=15.h5"
model = load_model(model_path)
dir_path = "D:/Rain/text/Python/MA_IIIT/models/vgg9/directions/vgg9_sgd_lr=0.1_bs=128_wd=0.0_epochs=15_weights_2D.h5"
f = h5py.File(dir_path, 'w')
tf.random.set_seed(123)
set_y = True
xdirection = creat_random_direction(model)
h5_util.write_list(f, 'xdirection', xdirection)
if set_y:
ydirection = creat_random_direction(model)
h5_util.write_list(f, 'ydirection', ydirection)
f.close()
print("direction file created")
a = 1
run_list = run_dict[model_type]
for run in run_list:
model_path = run["model_path"]
add_aug = run["add_aug"]
aug_pol = run["aug_pol"]
model.load_weights(model_path)
w = direction.get_weights(model)
norm_dx = direction.normalize_directions_for_weights(dx, w)
norm_dy = direction.normalize_directions_for_weights(dy, w)
dir_path = model_path[:-3] + '_' + fig_type + '_' + str(
l_range[0]) + '_' + str(l_range[1]) + '_same.h5'
f = h5py.File(dir_path, 'w')
h5_util.write_list(f, 'xdirection', norm_dx)
h5_util.write_list(f, 'ydirection', norm_dy)
f.close()
if 'norm' in model_path:
pre_mode = 'norm'
elif 'scale' in model_path:
pre_mode = 'scale'
else:
raise Exception('Unknown pre_mode!')
for loss_key, dot_num in zip(loss_key_list, dot_num_list):
main(model_type,
model_path,
batch_size=batch_size,
dataset=dataset,
def main(model_type,
model_path,
batch_size = 128,
dataset = 'cifar10',
load_mode = 'tfrd',
L_A = [3, 5],
L_W = [1, 7],
pre_mode = 'norm',
add_aug = False,
aug_pol = 'cifar_auto',
l2_reg_rate= None,
fc_type = None,
dir_path = None,
fig_type = '1D',
dot_num = 11,
l_range = (-1, 1),
loss_key = 'train_loss',
add_reg = True,
):
try:
model = load_model(model_path, custom_objects=CUSTOM_OBJ)
except Exception as e:
model = build_model(model_type, dataset, fc_type=fc_type, l2_reg_rate=l2_reg_rate, L_A=L_A, L_W=L_W).model
model.load_weights(model_path)
if dir_path == None:
dir_path = model_path[:-3] + '_' + fig_type + '_' + str(l_range[0]) + '_' + str(l_range[1]) + '.h5'
if os.path.exists(dir_path):
print("Direction file is already created.")
f = h5py.File(dir_path, 'r')
if fig_type == '2D':
assert ('xdirection' in f.keys() and 'ydirection' in f.keys()), "Please setup x/y direction!"
set_y = True
elif fig_type == '1D':
assert 'xdirection' in f.keys(), "Please setup x direction!"
set_y = False
f.close()
else:
f = h5py.File(dir_path, 'w')
xdirection = direction.creat_random_direction(model)
h5_util.write_list(f, 'xdirection', xdirection)
if fig_type == '2D':
ydirection = direction.creat_random_direction(model)
h5_util.write_list(f, 'ydirection', ydirection)
set_y = True
else:
set_y = False
f.close()
print("Direction file created.")
if 'qn' not in model_type:
surf_path = dir_path[:-3] + '_surface' + '_' + str(dot_num) + '_' + loss_key + '_add_reg=' + str(add_reg) +'.h5'
else:
surf_path = dir_path[:-3] + '_surface' + '_' + str(dot_num) + '_' + loss_key +'.h5'
w = direction.get_weights(model)
d = evaluation.load_directions(dir_path)
evaluation.setup_surface_file(surf_path, dir_path, set_y, num=dot_num, l_range=l_range)
if loss_key == 'train_loss':
acc_key = 'train_acc'
if not add_aug:
x_set, y_set, _, _ = data_loader.load_data(dataset, load_mode=load_mode)
x_mean = np.mean(x_set).astype('float32')
x_std = np.std(x_set).astype('float32')
#x_set = (x_set.astype('float32') - x_mean) / (x_std + 1e-7)
x_set = data_generator.preprocess_input(x_set, x_mean, x_std, mode=pre_mode)
else:
print("Load temp dataset.")
temp_file_path = data_generator.set_temp_dataset(dataset, load_mode, aug_pol, pre_mode=pre_mode)
x_set, y_set = data_generator.load_temp_dataset(temp_file_path)
print("Temp dataset loaded.")
#if os.path.exists(temp_file_path):
# os.remove(temp_file_path)
data_generator.remove_temp_dataset(temp_file_path)
elif loss_key == 'test_loss':
acc_key = 'test_acc'
x_train, _, x_set, y_set= data_loader.load_data(dataset, load_mode=load_mode)
x_mean = np.mean(x_train).astype('float32')
x_std = np.std(x_train).astype('float32')
#x_set = (x_set.astype('float32') - x_mean) / (x_std + 1e-7)
x_set = data_generator.preprocess_input(x_set, x_mean, x_std, mode=pre_mode)
else:
raise Exception("Unknown loss key: %s" % (loss_key))
evaluation.crunch(surf_path, model, model_type, w, d, x_set, y_set, loss_key, acc_key, batch_size=batch_size, add_reg=add_reg, L_A=L_A, L_W=L_W)
'''
if fig_type == '1D':
plot_1D.plot_1d_loss_err(surf_path, xmin=l_range[0], xmax=l_range[1], loss_max=5, log=False, show=False)
elif fig_type == '2D':
plot_2D.plot_2d_contour(surf_path, surf_name=loss_key, vmin=0.1, vmax=10, vlevel=0.5, show=False)
'''
return surf_path
