def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print( '-------------------------------------------------------------------') print('setup_direction') print( '-------------------------------------------------------------------') # Setup env for preventing lock on h5py file for newer h5py versions # added due to the commit https://github.com/tomgoldstein/loss-landscape/pull/28/files os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print("%s is already set up" % dir_file) return f.close() # Create the plotting directions # 下面介绍中, xdir 即 xdirection, ydir 即 ydirection # 第一层选择: 两个模型, or 一个模型, or 三个模型 # 第二层选择: 画1d 还是 2d # 两个模型 + 只画 x: 只有 xdir= model2 - model1 # (少见) 两个模型 + 画x,y: xdir = model2 - model 1, 但另一个方向ydirection 是随机 # (少见) 一个模型 + 只画 x: 只有 xdir = random. # 一个模型 + 画 x, y: xdir 和 ydir 都是随机 # 三个模型 + 画 x, y: xdir = model2 - model1, ydir = model3 - model 1. f = h5py.File(dir_file, 'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: # If extra model is provided, then only check xdirection net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: # If no extra file, then use a random direction. Use filter normalization by default xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: # If we want to draw 2d plots: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created: %s" % dir_file)
def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print( '-------------------------------------------------------------------') print('setup_direction') print( '-------------------------------------------------------------------') # Setup env for preventing lock on h5py file for newer h5py versions os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print("%s is already setted up" % dir_file) return f.close() # Create the plotting directions f = h5py.File(dir_file, 'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print("direction file created: %s" % dir_file)
def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos'): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model model_files: the checkpoint files dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print( 'The projection file exists! No projection is performed unless %s is deleted' % proj_file) return proj_file # read directions and convert them to vectors directions = net_plotter.load_directions(dir_file) dx = nplist_to_tensor(directions[0]) dy = nplist_to_tensor(directions[1]) xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = tensorlist_to_tensor(d) x, y = project_2D(d, dx, dy, proj_method) print("%s (%.4f, %.4f)" % (model_file, x, y)) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
def convert_matlab_pca_data(args, direction_matlab_name, direction_python_name): # class ARGS: # dataset='cifar10' # model='resnet56' # model_folder='folders for models to be projected' # dir_type='weights' # ignore='biasbn' # prefix='model_' # suffix='.t7' # start_epoch=0 # max_epoch=500 # save_epoch=1 # args = ARGS() # args.model_folder = model_folder # args.model = model last_model_file = args.model_folder + '/' + args.prefix + str( args.max_epoch) + args.suffix net = model_loader.load(args.dataset, args.model, last_model_file) w = net_plotter.get_weights(net) # read in matlab pca results f = h5py.File(direction_matlab_name, 'r') fpy = h5py.File(direction_python_name, 'w') fpy['explained_variance_ratio_'] = np.array(f['explained_variance_ratio_']) fpy['explained_variance_'] = np.array(f['explained_variance_']) pc1 = np.array(f['directionx']) pc2 = np.array(f['directiony']) f.close() # convert vectorized directions to the same shape as models to save in h5 file. # import pdb; pdb.set_trace() if args.dir_type == 'weights': xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) elif args.dir_type == 'states': xdirection = npvec_to_tensorlist(pc1, s) ydirection = npvec_to_tensorlist(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) # import pdb; pdb.set_trace() h5_util.write_list(fpy, 'xdirection', xdirection) h5_util.write_list(fpy, 'ydirection', ydirection) fpy.close() print('PCA directions saved in: %s' % direction_python_name)
def setup_direction(args, dir_file, net): """ Setup the h5 file to store the directions. - xdirection, ydirection: The pertubation direction added to the mdoel. The direction is a list of tensors. """ print('-------------------------------------------------------------------') print('setup_direction') print('-------------------------------------------------------------------') # Skip if the direction file already exists if exists(dir_file): f = h5py.File(dir_file, 'r') if (args.y and 'ydirection' in f.keys()) or 'xdirection' in f.keys(): f.close() print ("%s is already setted up" % dir_file) return f.close() # Create the plotting directions f = h5py.File(dir_file,'w') # create file, fail if exists if not args.dir_file: print("Setting up the plotting directions...") if args.model_file2: net2 = model_loader.load(args.dataset, args.model, args.model_file2) xdirection = create_target_direction(net, net2, args.dir_type) else: xdirection = create_random_direction(net, args.dir_type, args.xignore, args.xnorm) h5_util.write_list(f, 'xdirection', xdirection) if args.y: if args.same_dir: ydirection = xdirection elif args.model_file3: net3 = model_loader.load(args.dataset, args.model, args.model_file3) ydirection = create_target_direction(net, net3, args.dir_type) else: ydirection = create_random_direction(net, args.dir_type, args.yignore, args.ynorm) h5_util.write_list(f, 'ydirection', ydirection) f.close() print ("direction file created: %s" % dir_file)
def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos'): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model model_files: the checkpoint files dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print('The projection file exists! No projection is performed unless %s is deleted' % proj_file) return proj_file # read directions and convert them to vectors directions = net_plotter.load_directions(dir_file) dx = nplist_to_tensor(directions[0]) dy = nplist_to_tensor(directions[1]) xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = tensorlist_to_tensor(d) x, y = project_2D(d, dx, dy, proj_method) print ("%s (%.4f, %.4f)" % (model_file, x, y)) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
args.ymin, args.ymax, args.ynum = (None, None, None) if args.y: args.ymin, args.ymax, args.ynum = [ float(a) for a in args.y.split(':') ] assert args.ymin and args.ymax and args.ynum, \ 'You specified some arguments for the y axis, but not all' except: raise Exception( 'Improper format for x- or y-coordinates. Try something like -1:1:51' ) #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- net = model_loader.load(args.dataset, args.model, args.model_file) w = net_plotter.get_weights(net) # initial parameters s = copy.deepcopy( net.state_dict()) # deepcopy since state_dict are references if args.ngpu > 1: # data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file)
args.ymin, args.ymax, args.ynum = (None, None, None) if args.y: args.ymin, args.ymax, args.ynum = [ float(a) for a in args.y.split(':') ] assert args.ymin and args.ymax and args.ynum, \ 'You specified some arguments for the y axis, but not all' except: raise Exception( 'Improper format for x- or y-coordinates. Try something like -1:1:51' ) #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- net = model_loader.load(args.dataset, args.model, args.model_file, args.parallel, args.auxiliary) w = net_plotter.get_weights(net) # initial parameters s = copy.deepcopy( net.state_dict()) # deepcopy since state_dict are references if args.ngpu > 1: # data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file)
def main(): #-------------------------------------------------- # Parse input arguments #-------------------------------------------------- parser = argparse.ArgumentParser( description= 'Generation and evaluation of VGG-ANN crafted adversarial attack on the CIFAR10 or CIFAR100 dataset', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--seed', default=2, type=int, help='Random seed') parser.add_argument('--dataset', default='CIFAR10', type=str, help='{CIFAR10, CIFAR100} dataset type') parser.add_argument('--arch', default='VGG5', type=str, help='{VGG5, VGG11} network architecture') parser.add_argument('--log', default=None, type=str, help='Log file name') #------------------------------------------------------- # dataset parameters #-------------------------------------------------------- parser.add_argument( '--mean', default='0.5,0.5,0.5', type=str, help='mean of each channel of image in the format mean1,mean2,mean3') parser.add_argument( '--std', default='0.5,0.5,0.5', type=str, help='std of each channel of image in the format std1,std2,std3') parser.add_argument('--batch_size', default=64, type=int, help='Batch size') #------------------------------------------------------ # SNN parameters #------------------------------------------------------- parser.add_argument('--timesteps', default=100, type=int, help='Total number of time steps') parser.add_argument('--leak_mem', default=0.99, type=float, help='Leak in membrane potential for LIF neuron') #------------------------------------------------------- # Adversarial attack parameters #--------------------------------------------------------- parser.add_argument('--epsilon', default=8, type=int, help='Attack intensity. Possible values: 2,4,6,8,16') parser.add_argument( '--eps_iter', default=2, type=int, help= 'Attack intensity per iteration for PGD attack. Possible values:1,2') parser.add_argument('--pgd_steps', default=7, type=int, help='#steps in pgd. Possible values: 7, 10, 20, 40') parser.add_argument( '--rand_init', default=1, type=int, help= 'randomly perturb the original image in l-inf ball of epsilon for pgd. Possible values: 1, 0' ) parser.add_argument( '--source', default='ann', type=str, help='source network. Possible values: ann|snnconv|snnbp') parser.add_argument( '--target', default='ann', type=str, help='target network. Possible values: ann|snnconv|snnbp') parser.add_argument('--attack', default='fgsm', type=str, help='attack category. Possible values: fgsm|pgd') parser.add_argument( '--type', default='wb', type=str, help='Whitebox or Blackbox attack. possible values: wb|bb') parser.add_argument('--targeted', default='False', type=str, help='True|False') global args args = parser.parse_args() #-------------------------------------------------- # Initialize seed #-------------------------------------------------- seed = args.seed np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) #----------------------------------------------------- # dataset and architecture #----------------------------------------------------- dataset = args.dataset architecture = args.arch #----------------------------------------------------- # batch size #---------------------------------------------------- batch_size = args.batch_size #----------------------------------------------------- # Adversarial attack parameters #----------------------------------------------------- model_source_name = args.source model_target_name = args.target attack = args.attack targeted = args.targeted epsilon = args.epsilon eps_iter = args.eps_iter num_iter = args.pgd_steps rand_init = args.rand_init attack_type = args.type clip_min = 0.0 clip_max = 1.0 #--------------------------------------------------- # log file #---------------------------------------------------- logfile_name = dataset.lower() + architecture.lower( ) + '_' + model_source_name + '_crafted_on_' + model_target_name + '_' + attack_type + '_' + attack + '.log' f = open(logfile_name, 'a', buffering=1) #-------------------------------------------------- # Load the CIFAR10 dataset #-------------------------------------------------- data_root = expanduser("~") # These are the mean and std values used to train the models args.mean1, args.mean2, args.mean3 = [ float(a) for a in args.mean.split(',') ] args.std1, args.std2, args.std3 = [float(a) for a in args.std.split(',')] mean = [args.mean1, args.mean2, args.mean3] std = [args.std1, args.std2, args.std3] transform_test = transforms.Compose([transforms.ToTensor()]) if dataset == 'CIFAR10': testset = datasets.CIFAR10(root=data_root, train=False, download=True, transform=transform_test) testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, drop_last=True) #-------------------------------------------------- # Instantiate the Source & Target model #-------------------------------------------------- model_dir = os.getcwd() # map between model name and model_file directory model_source_dir = { 'ann_wb': model_dir + '/ann_checkpoint1.pt', 'ann_bb': model_dir + '/ann_checkpoint2.pt', 'snnconv_wb': model_dir + '/ann_checkpoint1.pt', 'snnconv_bb': model_dir + '/ann_checkpoint2.pt', 'snnbp_wb': model_dir + '/snnbp_checkpoint1.pt', 'snnbp_bb': model_dir + '/snnbp_checkpoint2.pt' } # The threshold values of SNN models. Two sets of thresholds corresponding to two separately initialized models snn_thresholds = [[20.76, 2.69, 2.33, 0.28, 1.14], [20.86, 2.74, 2.21, 0.38, 0.99]] # VGG5 # wb: whitebox, bb: blackbox if attack_type == 'wb': threshold_id = 0 elif attack_type == 'bb': threshold_id = 1 model_source = model_loader.load( model_source_name, batch_size, model_source_dir[model_source_name + '_' + attack_type], snn_thresholds[threshold_id][0:]) model_target = model_loader.load( model_target_name, batch_size, model_source_dir[model_target_name + '_wb'], snn_thresholds[0][0:]) # Used during analyzing the variation of number of timesteps and leak factor # if model_target.module.type == 'SNN': # model_target.module.timesteps_init(args.timesteps) # if model_target_name == 'snnbp': # model_target.module.leak_init(args.leak_mem) #-------------------------------------------------- # Initialize the loss function #-------------------------------------------------- loss_func = nn.CrossEntropyLoss().cuda() # Print the simulation parameters print('Batch size : {}'.format(batch_size)) print('{}-crafted attack on {}:{}'.format(model_source_name, model_target_name, attack_type)) print('Attack category: {}'.format(attack)) print('epsilon:{}|targeted:{}'.format(epsilon, targeted)) if attack == 'pgd': print('epsilon_iter:{}|num_iter:{}'.format(eps_iter, num_iter)) print('*********************************************************') #--------------------------------------------------------------------- # Adversarial attack generation and evaluation #-------------------------------------------------------------------- count_clean = 0 # keeps track of the correct predictions for clean input count_adv = 0 # keeps track of the correct predictions for the adversarial input # Instantiate the attack model if attack == 'fgsm': attack_model = FastGradientSign(epsilon * 1.0 / 255.0, clip_min, clip_max, targeted) elif attack == 'pgd': attack_model = ProjectedGradientDescent(num_iter, epsilon * 1.0 / 255.0, eps_iter * 1.0 / 255.0, clip_min, clip_max, targeted, rand_init, seed) for batch_id, (images, labels) in enumerate(testloader): images, labels = images.cuda(), labels.cuda() # Calculate the clean test accuracy of the target model acc, loss = eval_minibatch(model_target, images, labels, mean, std, loss_func) count_clean += acc # create target label in case of 'targeted' attack if targeted == 'False': target_labels = labels else: target_labels = random_targets(10, labels.cpu(), seed) target_labels = target_labels.cuda() # Generate adversarial example images_adv = attack_model.generate(images, labels, model_source, loss_func, mean, std) # Calculate the adv test accuracy of the target model acc, loss = eval_minibatch(model_target, images_adv, target_labels, mean, std, loss_func) count_adv += acc # Print the clean and adv count if (batch_id != 0) and (batch_id % 2 == 0): print('Minibatch : {}'.format(batch_id)) print('clean/adv: {}/{}\n'.format(int(count_clean), int(count_adv))) # Adversarial loss = precision_clean - precision_adv precision_adv = float(count_adv) * 100 / len(testloader.dataset) precision_clean = float(count_clean) * 100 / len(testloader.dataset) print('Adversarial Loss: {:.3f}%\n'.format(precision_clean - precision_adv)) if attack == 'pgd': f.write( "\t {}:{}:{}|targeted:{}|eps_iter:{}, num_iter:{}--> adv loss = {:.3f}%\n" .format(model_source_name, model_target_name, attack_type, targeted, eps_iter, num_iter, precision_clean - precision_adv)) elif attack == 'fgsm': f.write("\t {}:{}:{}|targeted:{}|epsilon:{}--> adv loss = {:.3f}%\n". format(model_source_name, model_target_name, attack_type, targeted, epsilon, precision_clean - precision_adv)) f.close()
def project_trajectory(dir_file, w, s, dataset, model_name, model_files, dir_type='weights', proj_method='cos', data_parallel=False): """ Project the optimization trajectory onto the given two directions. Args: dir_file: the h5 file that contains the directions w: weights of the final model s: states of the final model model_name: the name of the model save_epoch: the checkpoint frequency dir_type: the type of the direction, weights or states proj_method: cosine projection Returns: proj_file: the projection filename """ proj_file = dir_file + '_proj_' + proj_method + '.h5' if os.path.exists(proj_file): print(proj_file + ' exits! No projection is performed.') return proj_file # read directions and convert them to vectors f = h5py.File(dir_file, 'r') directions = net_plotter.load_directions(f) dx = list_to_vec(directions[0]) dy = list_to_vec(directions[1]) f.close() xcoord, ycoord = [], [] for model_file in model_files: net2 = model_loader.load(dataset, model_name, model_file, data_parallel) if dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) d = weights_to_vec(d) x, y = project_2D(d, dx, dy, proj_method) print(model_file, x, y) xcoord.append(x) ycoord.append(y) f = h5py.File(proj_file, 'w') f['proj_xcoord'] = np.array(xcoord) f['proj_ycoord'] = np.array(ycoord) f.close() return proj_file
from flask import Flask, request from flask import jsonify from model_loader import load model_name = "model.bin" model = load(model_name) app = Flask(__name__) @app.route('/vec/<term>') def vec(term): return jsonify(model[term].tolist()) @app.route('/vecs', methods=['POST']) def vecs(): if request.method == 'POST': terms = request.get_json() w2vecs = [model[term].tolist() for term in terms] return jsonify(w2vecs) return 'Only post request allowed'
# in case of multiple GPUs per node, set the GPU to use for each rank if args.cuda: if not torch.cuda.is_available(): raise Exception( 'User selected cuda option, but cuda is not available on this machine' ) gpu_count = torch.cuda.device_count() print('Use GPU %d of %d GPUs on %s' % (torch.cuda.current_device(), gpu_count, socket.gethostname())) for epoch in range(0, args.max_epoch, 5): #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- net = model_loader.load( args.dataset, args.model, args.model_folder + '/model_' + str(epoch) + '.t7') if args.ngpu > 1: # data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data', train=True, download=True)
args.ymin, args.ymax, args.ynum = [ float(a) for a in args.y.split(':') ] assert args.ymin and args.ymax and args.ynum, \ 'You specified some arguments for the y axis, but not all' except: raise Exception( 'Improper format for x- or y-coordinates. Try something like -1:1:51' ) #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- net = model_loader.load(args.dataset, args.model, args.model_file, data_parallel=args.data_parallel, num_blocks=args.num_blocks) w = net_plotter.get_weights(net) # initial parameters s = copy.deepcopy( net.state_dict()) # deepcopy since state_dict are references if args.ngpu > 1: # data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net)
s1_BCH = pd.read_csv('BCHs1_no_ask.csv').values s2_BCH = pd.read_csv('BCHs2_no_ask.csv').values s3_BCH = pd.read_csv('BCHs3_no_ask.csv').values s1_ETH = pd.read_csv('ETHs1_no_ask.csv').values s2_ETH = pd.read_csv('ETHs2_no_ask.csv').values s3_ETH = pd.read_csv('ETHs3_no_ask.csv').values s1_LTC = pd.read_csv('LTCs1_no_ask.csv').values s2_LTC = pd.read_csv('LTCs2_no_ask.csv').values s3_LTC = pd.read_csv('LTCs3_no_ask.csv').values # pre-load model to speed up gru_models = {} for asset in assets: gru_models[asset] = model_loader.load(asset) upper_bound = [516.77, 6546.40, 224.39, 58.64] lower_bound = [486.58, 6481.08, 218.36, 56.93] def predict_dpi(x, s): """Predict the average price change Δp_i, 1 <= i <= 3. Args: x: A numpy array of floats representing previous prices. s: A 2-dimensional numpy array generated by choose_effective_centers(). Returns: A big float representing average price change Δp_i. """
def setup_PCA_directions(args, model_files, w, s): """ Find PCA directions for the optimization path from the initial model to the final trained model. Returns: dir_name: the h5 file that stores the directions. """ # Name the .h5 file that stores the PCA directions. folder_name = args.model_folder + '/PCA_' + args.dir_type if args.ignore: folder_name += '_ignore=' + args.ignore folder_name += '_save_epoch=' + str(args.save_epoch) os.system('mkdir ' + folder_name) dir_name = folder_name + '/directions.h5' # skip if the direction file exists if os.path.exists(dir_name): f = h5py.File(dir_name, 'a') if 'explained_variance_' in f.keys(): f.close() return dir_name # load models and prepare the optimization path matrix matrix = [] for model_file in model_files: print (model_file) net2 = model_loader.load(args.dataset, args.model, model_file) if args.dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif args.dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(d) d = tensorlist_to_tensor(d) matrix.append(d.numpy()) # Perform PCA on the optimization path matrix print ("Perform PCA on the models") pca = PCA(n_components=2) pca.fit(np.array(matrix)) pc1 = np.array(pca.components_[0]) pc2 = np.array(pca.components_[1]) print("angle between pc1 and pc2: %f" % cal_angle(pc1, pc2)) print("pca.explained_variance_ratio_: %s" % str(pca.explained_variance_ratio_)) # convert vectorized directions to the same shape as models to save in h5 file. if args.dir_type == 'weights': xdirection = npvec_to_tensorlist(pc1, w) ydirection = npvec_to_tensorlist(pc2, w) elif args.dir_type == 'states': xdirection = npvec_to_tensorlist(pc1, s) ydirection = npvec_to_tensorlist(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) f = h5py.File(dir_name, 'w') h5_util.write_list(f, 'xdirection', xdirection) h5_util.write_list(f, 'ydirection', ydirection) f['explained_variance_ratio_'] = pca.explained_variance_ratio_ f['singular_values_'] = pca.singular_values_ f['explained_variance_'] = pca.explained_variance_ f.close() print ('PCA directions saved in: %s' % dir_name) return dir_name
# if int(epoch) == 150 or int(epoch) == 225 or int(epoch) == 275: # lr *= args.lr_decay # for param_group in optimizer.param_groups: # param_group['lr'] *= args._lr_decay f.close() sio.savemat('trained_nets/' + save_folder + '/' + args.model + '_gradient_noise.mat', mdict={'training_history': training_history,'testing_history': testing_history,'train_noise_norm': noise_norm_history_TRAIN,'weight_grad_history': weight_grad_history}, ) #-------------------------------------------------------------------------- # Load weights and save them in a mat file (temporal unit: epoch) #-------------------------------------------------------------------------- all_weights = [] for i in range(0,args.epochs+1,args.save_epoch): model_file = 'trained_nets/' + save_folder + '/' + 'model_' + str(i) + '.t7' net = ml.load('cifar10', args.model, model_file) w = net_plotter.get_weights(net) # initial parameters #s = copy.deepcopy(net.state_dict()) # deepcopy since state_dict are references #import pdb; pdb.set_trace() for j in range(len(w)): w[j] = w[j].cpu().numpy() all_weights.append(w) sio.savemat('trained_nets/' + save_folder + '/' + args.model + 'all_weights.mat', mdict={'weight': all_weights}, )
f = open('trained_nets/' + save_folder + '/log.out', 'a') trainloader, testloader = dataloader.get_data_loaders(args) if args.label_corrupt_prob and not args.resume_model: torch.save(trainloader, 'trained_nets/' + save_folder + '/trainloader.dat') torch.save(testloader, 'trained_nets/' + save_folder + '/testloader.dat') # Model if args.resume_model: # Load checkpoint. print('==> Resuming from checkpoint..') checkpoint = torch.load(args.resume_model) net = model_loader.load(args.model) net.load_state_dict(checkpoint['state_dict']) start_epoch = checkpoint['epoch'] + 1 else: net = model_loader.load(args.model) print(net) init_params(net) if args.ngpu > 1: net = torch.nn.DataParallel(net) criterion = nn.CrossEntropyLoss() if args.loss_name == 'mse': criterion = nn.MSELoss() if use_cuda:
# Check plotting resolution #-------------------------------------------------------------------------- try: args.xmin, args.xmax, args.xnum = [float(a) for a in args.x.split(':')] args.ymin, args.ymax, args.ynum = (None, None, None) if args.y: args.ymin, args.ymax, args.ynum = [float(a) for a in args.y.split(':')] assert args.ymin and args.ymax and args.ynum, \ 'You specified some arguments for the y axis, but not all' except: raise Exception('Improper format for x- or y-coordinates. Try something like -1:1:51') #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- net = model_loader.load(args.dataset, args.model, args.model_file) w = net_plotter.get_weights(net) # initial parameters s = copy.deepcopy(net.state_dict()) # deepcopy since state_dict are references if args.ngpu > 1: # data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0:
def setup_PCA_directions(args, model_files): """ Find PCA directions for the optimization path from the initial model to the final trained model. Returns: dir_name: the h5 file that stores the directions. """ # Name the .h5 file that stores the PCA directions. folder_name = args.model_folder + '/PCA_' + args.dir_type if args.ignore: folder_name += '_ignore=' + args.ignore folder_name += '_save_epoch=' + str(args.save_epoch) os.system('mkdir ' + folder_name) dir_name = folder_name + '/directions.h5' # skip if the direction file exists if os.path.exists(dir_name): f = h5py.File(dir_name, 'a') if 'explained_variance_' in f.keys(): f.close() return dir_name # load models and prepare the optimization path matrix matrix = [] for model_file in model_files: print(model_file) net2 = model_loader.load(args.dataset, args.model, model_file, args.data_parallel) if args.dir_type == 'weights': w2 = net_plotter.get_weights(net2) d = net_plotter.get_diff_weights(w, w2) elif args.dir_type == 'states': s2 = net2.state_dict() d = net_plotter.get_diff_states(s, s2) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(d) d = weights_to_vec(d) matrix.append(d.numpy()) # Perform PCA on the optimization path matrix print("Perform PCA on the models") pca = PCA(n_components=2) pca.fit(np.array(matrix)) pc1 = np.array(pca.components_[0]) pc2 = np.array(pca.components_[1]) print("angle between pc1 and pc2: " + str(cal_angle(pc1, pc2))) print(pca.explained_variance_ratio_) # convert vectorized directions to the same shape as models to save in h5 file. if args.dir_type == 'weights': xdirection = vec_to_weights(pc1, w) ydirection = vec_to_weights(pc2, w) elif args.dir_type == 'states': xdirection = vec_to_states(pc1, s) ydirection = vec_to_states(pc2, s) if args.ignore == 'biasbn': net_plotter.ignore_biasbn(xdirection) net_plotter.ignore_biasbn(ydirection) f = h5py.File(dir_name, 'w') h5_util.write_list(f, 'xdirection', xdirection) h5_util.write_list(f, 'ydirection', ydirection) f['explained_variance_ratio_'] = pca.explained_variance_ratio_ f['singular_values_'] = pca.singular_values_ f['explained_variance_'] = pca.explained_variance_ f.close() print('PCA directions saved in ' + dir_name) return dir_name
states (include BN.running_mean/var)""") parser.add_argument('--ignore', default='', help='ignore bias and BN paras: biasbn (no bias or bn)') parser.add_argument('--prefix', default='model_', help='prefix for the checkpint model') parser.add_argument('--suffix', default='.pth', help='prefix for the checkpint model') parser.add_argument('--start_epoch', default=0, type=int, help='min index of epochs') parser.add_argument('--max_epoch', default=300, type=int, help='max number of epochs') parser.add_argument('--save_epoch', default=1, type=int, help='save models every few epochs') parser.add_argument('--dir_file', default='', help='load the direction file for projection') parser.add_argument('--data_parallel', action='store_true', default=False, help='the models are saved in data_parallel format') args = parser.parse_args() # load the final model last_model_file = args.model_folder + '/' + args.prefix + str(args.max_epoch) + args.suffix net = model_loader.load(args.dataset, args.model, last_model_file, args.data_parallel) w = net_plotter.get_weights(net) s = net.state_dict() # collect models to be projected model_files = [] for epoch in range(args.start_epoch, args.max_epoch + args.save_epoch, args.save_epoch): model_file = args.model_folder + '/' + args.prefix + str(epoch) + args.suffix assert os.path.exist(model_file), 'model %s does not exist' % model_file model_files.append(model_file) # load or create projection directions if args.dir_file: dir_file = args.dir_file else: dir_file = setup_PCA_directions(args, model_files)