def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(
'The projection file exists! No projection is performed unless %s is deleted'
% proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def project_trajectory(dir_file, w, s, dataset, model_name, model_files,
dir_type='weights', proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print('The projection file exists! No projection is performed unless %s is deleted' % proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print ("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def setup_PCA_directions(args, model_files):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print(model_file)
net2 = model_loader.load(args.dataset, args.model, model_file,
args.data_parallel)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = weights_to_vec(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: " + str(cal_angle(pc1, pc2)))
print(pca.explained_variance_ratio_)
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = vec_to_weights(pc1, w)
ydirection = vec_to_weights(pc2, w)
elif args.dir_type == 'states':
xdirection = vec_to_states(pc1, s)
ydirection = vec_to_states(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print('PCA directions saved in ' + dir_name)
return dir_name
def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos',
data_parallel=False):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
save_epoch: the checkpoint frequency
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(proj_file + ' exits! No projection is performed.')
return proj_file
# read directions and convert them to vectors
f = h5py.File(dir_file, 'r')
directions = net_plotter.load_directions(f)
dx = list_to_vec(directions[0])
dy = list_to_vec(directions[1])
f.close()
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file,
data_parallel)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = weights_to_vec(d)
x, y = project_2D(d, dx, dy, proj_method)
print(model_file, x, y)
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def setup_PCA_directions(args, model_files, w, s):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
folder_name = args.model_folder + '/PCA_' + args.dir_type
if args.ignore:
folder_name += '_ignore=' + args.ignore
folder_name += '_save_epoch=' + str(args.save_epoch)
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print (model_file)
net2 = model_loader.load(args.dataset, args.model, model_file)
if args.dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif args.dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print ("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_))
# convert vectorized directions to the same shape as models to save in h5 file.
if args.dir_type == 'weights':
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
elif args.dir_type == 'states':
xdirection = npvec_to_tensorlist(pc1, s)
ydirection = npvec_to_tensorlist(pc2, s)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print ('PCA directions saved in: %s' % dir_name)
return dir_name
def setup_othermodels_PCA_directions(args, ignore_embedding, model_files, w):
"""
Find PCA directions for the optimization path from the initial model
to the final trained model.
Returns:
dir_name: the h5 file that stores the directions.
"""
# Name the .h5 file that stores the PCA directions.
# Name the .h5 file that stores the PCA directions.
folder_name = 'fairseq_master' + '/' + args.save_dir + '/PCA_'
folder_name += 'lr=' + str(args.lr[0])
folder_name += '_optimier=' + str(args.optimizer)
folder_name += '_ignore_embedding=' + str(ignore_embedding)
if args.ignore:
folder_name += '_ignoreBN'
os.system('mkdir ' + folder_name)
dir_name = folder_name + '/directions.h5'
# skip if the direction file exists
if os.path.exists(dir_name):
f = h5py.File(dir_name, 'a')
if 'explained_variance_' in f.keys():
f.close()
return dir_name
# load models and prepare the optimization path matrix
matrix = []
for model_file in model_files:
print(model_file)
state = torch.load(
model_file,
map_location=lambda s, l: torch.serialization.
default_restore_location(s, 'cpu'),
)
args2 = state['args']
args2.data = 'fairseq_master/' + args2.data
# task = tasks.setup_task(args2)
# model2 = task.build_model(args2)
# s2 = model2.state_dict()
s2 = state['model']
w2 = []
if ignore_embedding:
for key in s2:
if 'version' in key or '_float_tensor' in key or 'embed' in key:
s2[key].fill_(0)
w2.append(s2[key])
else:
for key in s2:
if 'version' in key or '_float_tensor' in key:
s2[key].fill_(0)
w2.append(s2[key])
d = net_plotter.get_diff_weights(w, w2)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(d)
d = tensorlist_to_tensor(d)
matrix.append(d.numpy())
# Perform PCA on the optimization path matrix
print("Perform PCA on the models")
pca = PCA(n_components=2)
pca.fit(np.array(matrix))
pc1 = np.array(pca.components_[0])
pc2 = np.array(pca.components_[1])
print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2))
print("pca.explained_variance_ratio_: %s" %
str(pca.explained_variance_ratio_))
xdirection = npvec_to_tensorlist(pc1, w)
ydirection = npvec_to_tensorlist(pc2, w)
if args.ignore == 'biasbn':
net_plotter.ignore_biasbn(xdirection)
net_plotter.ignore_biasbn(ydirection)
f = h5py.File(dir_name, 'w')
h5_util.write_list(f, 'xdirection', xdirection)
h5_util.write_list(f, 'ydirection', ydirection)
f['explained_variance_ratio_'] = pca.explained_variance_ratio_
f['singular_values_'] = pca.singular_values_
f['explained_variance_'] = pca.explained_variance_
f.close()
print('PCA directions saved in: %s' % dir_name)
return dir_name
def project_othermodels_trajectory(dir_file,
w,
model_files,
ignore_embedding,
proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(
'The projection file exists! No projection is performed unless %s is deleted'
% proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
state = torch.load(
model_file,
map_location=lambda s, l: torch.serialization.
default_restore_location(s, 'cpu'),
)
args2 = state['args']
args2.data = 'fairseq_master/' + args2.data
# task = tasks.setup_task(args2)
# model2 = task.build_model(args2)
# s2 = model2.state_dict()
s2 = state['model']
w2 = []
if ignore_embedding:
for key in s2:
if 'version' in key or '_float_tensor' in key or 'embed' in key:
s2[key].fill_(0)
w2.append(s2[key])
else:
for key in s2:
if 'version' in key or '_float_tensor' in key:
s2[key].fill_(0)
w2.append(s2[key])
d = net_plotter.get_diff_weights(w, w2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file