def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print( '-------------------------------------------------------------------') print('setup_direction') print( '-------------------------------------------------------------------') # Setup env for preventing lock on h5py file for newer h5py versions # added due to the commit https://github.com/tomgoldstein/loss-landscape/pull/28/files os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print("%s is already set up" % dir_file) return f.close() # Create the plotting directions # 下面介绍中, xdir 即 xdirection, ydir 即 ydirection # 第一层选择: 两个模型, or 一个模型, or 三个模型 # 第二层选择: 画1d 还是 2d # 两个模型 + 只画 x: 只有 xdir= model2 - model1 # (少见) 两个模型 + 画x,y: xdir = model2 - model 1, 但另一个方向ydirection 是随机 # (少见) 一个模型 + 只画 x: 只有 xdir = random. # 一个模型 + 画 x, y: xdir 和 ydir 都是随机 # 三个模型 + 画 x, y: xdir = model2 - model1, ydir = model3 - model 1. f = h5py.File(dir_file, 'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: # If extra model is provided, then only check xdirection net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: # If no extra file, then use a random direction. Use filter normalization by default xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: # If we want to draw 2d plots: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created: %s" % dir_file)
def convert_matlab_pca_data(args, direction_matlab_name, direction_python_name): # class ARGS: # dataset='cifar10' # model='resnet56' # model_folder='folders for models to be projected' # dir_type='weights' # ignore='biasbn' # prefix='model_' # suffix='.t7' # start_epoch=0 # max_epoch=500 # save_epoch=1 # args = ARGS() # args.model_folder = model_folder # args.model = model last_model_file = args.model_folder + '/' + args.prefix + str( args.max_epoch) + args.suffix net = model_loader.load(args.dataset, args.model, last_model_file) w = net_plotter.get_weights(net) # read in matlab pca results f = h5py.File(direction_matlab_name, 'r') fpy = h5py.File(direction_python_name, 'w') fpy['explained_variance_ratio_'] = np.array(f['explained_variance_ratio_']) fpy['explained_variance_'] = np.array(f['explained_variance_']) pc1 = np.array(f['directionx']) pc2 = np.array(f['directiony']) f.close() # convert vectorized directions to the same shape as models to save in h5 file. # import pdb; pdb.set_trace() if args.dir_type == 'weights': xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) elif args.dir_type == 'states': xdirection = npvec_to_tensorlist(pc1, s) ydirection = npvec_to_tensorlist(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) # import pdb; pdb.set_trace() h5_util.write_list(fpy, 'xdirection', xdirection) h5_util.write_list(fpy, 'ydirection', ydirection) fpy.close() print('PCA directions saved in: %s' % direction_python_name)
def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print( '-------------------------------------------------------------------') print('setup_direction') print( '-------------------------------------------------------------------') # Setup env for preventing lock on h5py file for newer h5py versions os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print("%s is already setted up" % dir_file) return f.close() # Create the plotting directions f = h5py.File(dir_file, 'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print('-------------------------------------------------------------------') print('setup_direction') print('-------------------------------------------------------------------') # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print ("%s is already setted up" % dir_file) return f.close() # Create the plotting directions f = h5py.File(dir_file,'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print ("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, model): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print( '-------------------------------------------------------------------') print('setup_direction') print( '-------------------------------------------------------------------') # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print("%s is already setted up" % dir_file) return f.close() # Create the plotting directions f = h5py.File(dir_file, 'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") xdirection = create_random_direction(model, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: if args.same_dir: ydirection = xdirection else: ydirection = create_random_direction(model, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created: %s" % dir_file)
def setup_PCA_directions(args, model_files, w, s): """ Find PCA directions for the optimization path from the initial model to the final trained model. Returns: dir_name: the h5 file that stores the directions. """ # Name the .h5 file that stores the PCA directions. folder_name = args.model_folder + '/PCA_' + args.dir_type if args.ignore: folder_name += '_ignore=' + args.ignore folder_name += '_save_epoch=' + str(args.save_epoch) os.system('mkdir ' + folder_name) dir_name = folder_name + '/directions.h5' # skip if the direction file exists if os.path.exists(dir_name): f = h5py.File(dir_name, 'a') if 'explained_variance_' in f.keys(): f.close() return dir_name # load models and prepare the optimization path matrix matrix = [] for model_file in model_files: print (model_file) net2 = model_loader.load(args.dataset, args.model, model_file) if args.dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif args.dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(d) d = tensorlist_to_tensor(d) matrix.append(d.numpy()) # Perform PCA on the optimization path matrix print ("Perform PCA on the models") pca = PCA(n_components=2) pca.fit(np.array(matrix)) pc1 = np.array(pca.components_[0]) pc2 = np.array(pca.components_[1]) print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2)) print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_)) # convert vectorized directions to the same shape as models to save in h5 file. if args.dir_type == 'weights': xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) elif args.dir_type == 'states': xdirection = npvec_to_tensorlist(pc1, s) ydirection = npvec_to_tensorlist(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) f = h5py.File(dir_name, 'w') h5_util.write_list(f, 'xdirection', xdirection) h5_util.write_list(f, 'ydirection', ydirection) f['explained_variance_ratio_'] = pca.explained_variance_ratio_ f['singular_values_'] = pca.singular_values_ f['explained_variance_'] = pca.explained_variance_ f.close() print ('PCA directions saved in: %s' % dir_name) return dir_name
def setup_PCA_directions(args, model_files, w, s): """ Find PCA directions for the optimization path from the initial model to the final trained model. Returns: dir_name: the h5 file that stores the directions. """ # Name the .h5 file that stores the PCA directions. folder_name = args.model_folder + '/PCA_' + args.dir_type if args.ignore: folder_name += '_ignore=' + args.ignore folder_name += '_save_epoch=' + str(args.save_epoch) os.system('mkdir ' + folder_name) dir_name = folder_name + '/directions.h5' # skip if the direction file exists if os.path.exists(dir_name): f = h5py.File(dir_name, 'a') if 'explained_variance_' in f.keys(): f.close() return dir_name # load models and prepare the optimization path matrix matrix = [] for model_file in model_files: print(model_file) net2 = model_loader.load(args.dataset, args.model, model_file) if args.dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif args.dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(d) d = tensorlist_to_tensor(d) matrix.append(d.numpy()) # Perform PCA on the optimization path matrix print("Perform PCA on the models") pca = PCA(n_components=2) pca.fit(np.array(matrix)) pc1 = np.array(pca.components_[0]) pc2 = np.array(pca.components_[1]) print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2)) print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_)) # convert vectorized directions to the same shape as models to save in h5 file. if args.dir_type == 'weights': xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) elif args.dir_type == 'states': xdirection = npvec_to_tensorlist(pc1, s) ydirection = npvec_to_tensorlist(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) f = h5py.File(dir_name, 'w') h5_util.write_list(f, 'xdirection', xdirection) h5_util.write_list(f, 'ydirection', ydirection) f['explained_variance_ratio_'] = pca.explained_variance_ratio_ f['singular_values_'] = pca.singular_values_ f['explained_variance_'] = pca.explained_variance_ f.close() print('PCA directions saved in: %s' % dir_name) return dir_name
def setup_othermodels_PCA_directions(args, ignore_embedding, model_files, w): """ Find PCA directions for the optimization path from the initial model to the final trained model. Returns: dir_name: the h5 file that stores the directions. """ # Name the .h5 file that stores the PCA directions. # Name the .h5 file that stores the PCA directions. folder_name = 'fairseq_master' + '/' + args.save_dir + '/PCA_' folder_name += 'lr=' + str(args.lr[0]) folder_name += '_optimier=' + str(args.optimizer) folder_name += '_ignore_embedding=' + str(ignore_embedding) if args.ignore: folder_name += '_ignoreBN' os.system('mkdir ' + folder_name) dir_name = folder_name + '/directions.h5' # skip if the direction file exists if os.path.exists(dir_name): f = h5py.File(dir_name, 'a') if 'explained_variance_' in f.keys(): f.close() return dir_name # load models and prepare the optimization path matrix matrix = [] for model_file in model_files: print(model_file) state = torch.load( model_file, map_location=lambda s, l: torch.serialization. default_restore_location(s, 'cpu'), ) args2 = state['args'] args2.data = 'fairseq_master/' + args2.data # task = tasks.setup_task(args2) # model2 = task.build_model(args2) # s2 = model2.state_dict() s2 = state['model'] w2 = [] if ignore_embedding: for key in s2: if 'version' in key or '_float_tensor' in key or 'embed' in key: s2[key].fill_(0) w2.append(s2[key]) else: for key in s2: if 'version' in key or '_float_tensor' in key: s2[key].fill_(0) w2.append(s2[key]) d = net_plotter.get_diff_weights(w, w2) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(d) d = tensorlist_to_tensor(d) matrix.append(d.numpy()) # Perform PCA on the optimization path matrix print("Perform PCA on the models") pca = PCA(n_components=2) pca.fit(np.array(matrix)) pc1 = np.array(pca.components_[0]) pc2 = np.array(pca.components_[1]) print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2)) print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_)) xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) f = h5py.File(dir_name, 'w') h5_util.write_list(f, 'xdirection', xdirection) h5_util.write_list(f, 'ydirection', ydirection) f['explained_variance_ratio_'] = pca.explained_variance_ratio_ f['singular_values_'] = pca.singular_values_ f['explained_variance_'] = pca.explained_variance_ f.close() print('PCA directions saved in: %s' % dir_name) return dir_name
if __name__ == "__main__": gpus = tf.config.experimental.list_physical_devices( 'GPU') #must limit gpu memory growth for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) model_path = "D:/Rain/text/Python/MA_IIIT/models/vgg9/vgg9_sgd_lr=0.1_bs=128_wd=0.0_epochs=15.h5" model = load_model(model_path) dir_path = "D:/Rain/text/Python/MA_IIIT/models/vgg9/directions/vgg9_sgd_lr=0.1_bs=128_wd=0.0_epochs=15_weights_2D.h5" f = h5py.File(dir_path, 'w') tf.random.set_seed(123) set_y = True xdirection = creat_random_direction(model) h5_util.write_list(f, 'xdirection', xdirection) if set_y: ydirection = creat_random_direction(model) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created") a = 1
run_list = run_dict[model_type] for run in run_list: model_path = run["model_path"] add_aug = run["add_aug"] aug_pol = run["aug_pol"] model.load_weights(model_path) w = direction.get_weights(model) norm_dx = direction.normalize_directions_for_weights(dx, w) norm_dy = direction.normalize_directions_for_weights(dy, w) dir_path = model_path[:-3] + '_' + fig_type + '_' + str( l_range[0]) + '_' + str(l_range[1]) + '_same.h5' f = h5py.File(dir_path, 'w') h5_util.write_list(f, 'xdirection', norm_dx) h5_util.write_list(f, 'ydirection', norm_dy) f.close() if 'norm' in model_path: pre_mode = 'norm' elif 'scale' in model_path: pre_mode = 'scale' else: raise Exception('Unknown pre_mode!') for loss_key, dot_num in zip(loss_key_list, dot_num_list): main(model_type, model_path, batch_size=batch_size, dataset=dataset,
def main(model_type, model_path, batch_size = 128, dataset = 'cifar10', load_mode = 'tfrd', L_A = [3, 5], L_W = [1, 7], pre_mode = 'norm', add_aug = False, aug_pol = 'cifar_auto', l2_reg_rate= None, fc_type = None, dir_path = None, fig_type = '1D', dot_num = 11, l_range = (-1, 1), loss_key = 'train_loss', add_reg = True, ): try: model = load_model(model_path, custom_objects=CUSTOM_OBJ) except Exception as e: model = build_model(model_type, dataset, fc_type=fc_type, l2_reg_rate=l2_reg_rate, L_A=L_A, L_W=L_W).model model.load_weights(model_path) if dir_path == None: dir_path = model_path[:-3] + '_' + fig_type + '_' + str(l_range[0]) + '_' + str(l_range[1]) + '.h5' if os.path.exists(dir_path): print("Direction file is already created.") f = h5py.File(dir_path, 'r') if fig_type == '2D': assert ('xdirection' in f.keys() and 'ydirection' in f.keys()), "Please setup x/y direction!" set_y = True elif fig_type == '1D': assert 'xdirection' in f.keys(), "Please setup x direction!" set_y = False f.close() else: f = h5py.File(dir_path, 'w') xdirection = direction.creat_random_direction(model) h5_util.write_list(f, 'xdirection', xdirection) if fig_type == '2D': ydirection = direction.creat_random_direction(model) h5_util.write_list(f, 'ydirection', ydirection) set_y = True else: set_y = False f.close() print("Direction file created.") if 'qn' not in model_type: surf_path = dir_path[:-3] + '_surface' + '_' + str(dot_num) + '_' + loss_key + '_add_reg=' + str(add_reg) +'.h5' else: surf_path = dir_path[:-3] + '_surface' + '_' + str(dot_num) + '_' + loss_key +'.h5' w = direction.get_weights(model) d = evaluation.load_directions(dir_path) evaluation.setup_surface_file(surf_path, dir_path, set_y, num=dot_num, l_range=l_range) if loss_key == 'train_loss': acc_key = 'train_acc' if not add_aug: x_set, y_set, _, _ = data_loader.load_data(dataset, load_mode=load_mode) x_mean = np.mean(x_set).astype('float32') x_std = np.std(x_set).astype('float32') #x_set = (x_set.astype('float32') - x_mean) / (x_std + 1e-7) x_set = data_generator.preprocess_input(x_set, x_mean, x_std, mode=pre_mode) else: print("Load temp dataset.") temp_file_path = data_generator.set_temp_dataset(dataset, load_mode, aug_pol, pre_mode=pre_mode) x_set, y_set = data_generator.load_temp_dataset(temp_file_path) print("Temp dataset loaded.") #if os.path.exists(temp_file_path): # os.remove(temp_file_path) data_generator.remove_temp_dataset(temp_file_path) elif loss_key == 'test_loss': acc_key = 'test_acc' x_train, _, x_set, y_set= data_loader.load_data(dataset, load_mode=load_mode) x_mean = np.mean(x_train).astype('float32') x_std = np.std(x_train).astype('float32') #x_set = (x_set.astype('float32') - x_mean) / (x_std + 1e-7) x_set = data_generator.preprocess_input(x_set, x_mean, x_std, mode=pre_mode) else: raise Exception("Unknown loss key: %s" % (loss_key)) evaluation.crunch(surf_path, model, model_type, w, d, x_set, y_set, loss_key, acc_key, batch_size=batch_size, add_reg=add_reg, L_A=L_A, L_W=L_W) ''' if fig_type == '1D': plot_1D.plot_1d_loss_err(surf_path, xmin=l_range[0], xmax=l_range[1], loss_max=5, log=False, show=False) elif fig_type == '2D': plot_2D.plot_2d_contour(surf_path, surf_name=loss_key, vmin=0.1, vmax=10, vlevel=0.5, show=False) ''' return surf_path