def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(
'The projection file exists! No projection is performed unless %s is deleted'
% proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def project_trajectory(dir_file, w, s, dataset, model_name, model_files,
dir_type='weights', proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print('The projection file exists! No projection is performed unless %s is deleted' % proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print ("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
def project_trajectory(dir_file,
w,
s,
dataset,
model_name,
model_files,
dir_type='weights',
proj_method='cos',
data_parallel=False):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
save_epoch: the checkpoint frequency
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(proj_file + ' exits! No projection is performed.')
return proj_file
# read directions and convert them to vectors
f = h5py.File(dir_file, 'r')
directions = net_plotter.load_directions(f)
dx = list_to_vec(directions[0])
dy = list_to_vec(directions[1])
f.close()
xcoord, ycoord = [], []
for model_file in model_files:
net2 = model_loader.load(dataset, model_name, model_file,
data_parallel)
if dir_type == 'weights':
w2 = net_plotter.get_weights(net2)
d = net_plotter.get_diff_weights(w, w2)
elif dir_type == 'states':
s2 = net2.state_dict()
d = net_plotter.get_diff_states(s, s2)
d = weights_to_vec(d)
x, y = project_2D(d, dx, dy, proj_method)
print(model_file, x, y)
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
surf_file = name_surface_file(args, dir_file)
if rank == 0:
setup_surface_file(args, surf_file, dir_file)
# wait until master has setup the direction file and surface file
mpi.barrier(comm)
# load directions
d = net_plotter.load_directions(dir_file)
# calculate the consine similarity of the two directions
if len(d) == 2 and rank == 0:
similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]),
proj.nplist_to_tensor(d[1]))
print('cosine similarity between x-axis and y-axis: %f' % similarity)
#--------------------------------------------------------------------------
# Setup dataloader
#--------------------------------------------------------------------------
# download CIFAR10 if it does not exit
if rank == 0 and args.dataset == 'cifar10':
torchvision.datasets.CIFAR10(root=args.dataset + '/data',
train=True,
download=True)
def project_othermodels_trajectory(dir_file,
w,
model_files,
ignore_embedding,
proj_method='cos'):
"""
Project the optimization trajectory onto the given two directions.
Args:
dir_file: the h5 file that contains the directions
w: weights of the final model
s: states of the final model
model_name: the name of the model
model_files: the checkpoint files
dir_type: the type of the direction, weights or states
proj_method: cosine projection
Returns:
proj_file: the projection filename
"""
proj_file = dir_file + '_proj_' + proj_method + '.h5'
if os.path.exists(proj_file):
print(
'The projection file exists! No projection is performed unless %s is deleted'
% proj_file)
return proj_file
# read directions and convert them to vectors
directions = net_plotter.load_directions(dir_file)
dx = nplist_to_tensor(directions[0])
dy = nplist_to_tensor(directions[1])
xcoord, ycoord = [], []
for model_file in model_files:
state = torch.load(
model_file,
map_location=lambda s, l: torch.serialization.
default_restore_location(s, 'cpu'),
)
args2 = state['args']
args2.data = 'fairseq_master/' + args2.data
# task = tasks.setup_task(args2)
# model2 = task.build_model(args2)
# s2 = model2.state_dict()
s2 = state['model']
w2 = []
if ignore_embedding:
for key in s2:
if 'version' in key or '_float_tensor' in key or 'embed' in key:
s2[key].fill_(0)
w2.append(s2[key])
else:
for key in s2:
if 'version' in key or '_float_tensor' in key:
s2[key].fill_(0)
w2.append(s2[key])
d = net_plotter.get_diff_weights(w, w2)
d = tensorlist_to_tensor(d)
x, y = project_2D(d, dx, dy, proj_method)
print("%s (%.4f, %.4f)" % (model_file, x, y))
xcoord.append(x)
ycoord.append(y)
f = h5py.File(proj_file, 'w')
f['proj_xcoord'] = np.array(xcoord)
f['proj_ycoord'] = np.array(ycoord)
f.close()
return proj_file
#--------------------------------------------------------------------------
# Setup the direction file and the surface file
#--------------------------------------------------------------------------
dir_file = net_plotter.name_direction_file(args) # name the direction file
if rank == 0:
net_plotter.setup_direction(args, dir_file, net)
surf_file = name_surface_file(args, dir_file)
if rank == 0:
setup_surface_file(args, surf_file, dir_file)
# wait until master has setup the direction file and surface file
mpi4pytorch.barrier(comm)
# load directions
d = net_plotter.load_directions(dir_file)
# calculate the consine similarity of the two directions
if len(d) == 2 and rank == 0:
similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1]))
print('cosine similarity between x-axis and y-axis: %f' % similarity)
#--------------------------------------------------------------------------
# Setup dataloader
#--------------------------------------------------------------------------
# download CIFAR10 if it does not exit
if rank == 0 and args.dataset == 'cifar10':
torchvision.datasets.CIFAR10(root=args.dataset + '/data', train=True, download=True)
mpi4pytorch.barrier(comm)
trainloader, testloader = dataloader.load_dataset(args.dataset, args.datapath,
