def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos'): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model model_files: the checkpoint files dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print( 'The projection file exists! No projection is performed unless %s is deleted' % proj_file) return proj_file # read directions and convert them to vectors directions = net_plotter.load_directions(dir_file) dx = nplist_to_tensor(directions[0]) dy = nplist_to_tensor(directions[1]) xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = tensorlist_to_tensor(d) x, y = project_2D(d, dx, dy, proj_method) print("%s (%.4f, %.4f)" % (model_file, x, y)) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos'): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model model_files: the checkpoint files dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print('The projection file exists! No projection is performed unless %s is deleted' % proj_file) return proj_file # read directions and convert them to vectors directions = net_plotter.load_directions(dir_file) dx = nplist_to_tensor(directions[0]) dy = nplist_to_tensor(directions[1]) xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = tensorlist_to_tensor(d) x, y = project_2D(d, dx, dy, proj_method) print ("%s (%.4f, %.4f)" % (model_file, x, y)) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos', data_parallel=False): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model save_epoch: the checkpoint frequency dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print(proj_file + ' exits! No projection is performed.') return proj_file # read directions and convert them to vectors f = h5py.File(dir_file, 'r') directions = net_plotter.load_directions(f) dx = list_to_vec(directions[0]) dy = list_to_vec(directions[1]) f.close() xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file, data_parallel) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = weights_to_vec(d) x, y = project_2D(d, dx, dy, proj_method) print(model_file, x, y) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
#-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0: setup_surface_file(args, surf_file, dir_file) # wait until master has setup the direction file and surface file mpi.barrier(comm) # load directions d = net_plotter.load_directions(dir_file) # calculate the consine similarity of the two directions if len(d) == 2 and rank == 0: similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1])) print('cosine similarity between x-axis and y-axis: %f' % similarity) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if rank == 0 and args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data', train=True, download=True)
def project_othermodels_trajectory(dir_file, w, model_files, ignore_embedding, proj_method='cos'): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model model_files: the checkpoint files dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print( 'The projection file exists! No projection is performed unless %s is deleted' % proj_file) return proj_file # read directions and convert them to vectors directions = net_plotter.load_directions(dir_file) dx = nplist_to_tensor(directions[0]) dy = nplist_to_tensor(directions[1]) xcoord, ycoord = [], [] for model_file in model_files: state = torch.load( model_file, map_location=lambda s, l: torch.serialization. default_restore_location(s, 'cpu'), ) args2 = state['args'] args2.data = 'fairseq_master/' + args2.data # task = tasks.setup_task(args2) # model2 = task.build_model(args2) # s2 = model2.state_dict() s2 = state['model'] w2 = [] if ignore_embedding: for key in s2: if 'version' in key or '_float_tensor' in key or 'embed' in key: s2[key].fill_(0) w2.append(s2[key]) else: for key in s2: if 'version' in key or '_float_tensor' in key: s2[key].fill_(0) w2.append(s2[key]) d = net_plotter.get_diff_weights(w, w2) d = tensorlist_to_tensor(d) x, y = project_2D(d, dx, dy, proj_method) print("%s (%.4f, %.4f)" % (model_file, x, y)) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
#-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0: setup_surface_file(args, surf_file, dir_file) # wait until master has setup the direction file and surface file mpi4pytorch.barrier(comm) # load directions d = net_plotter.load_directions(dir_file) # calculate the consine similarity of the two directions if len(d) == 2 and rank == 0: similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1])) print('cosine similarity between x-axis and y-axis: %f' % similarity) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if rank == 0 and args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data', train=True, download=True) mpi4pytorch.barrier(comm) trainloader, testloader = dataloader.load_dataset(args.dataset, args.datapath,