def pruning_process(model): print("------------------------- Before pruning --------------------------------") criterion = nn.CrossEntropyLoss().cuda() accuracy = util.validate(args, test_loader, model, criterion) print("------------------------- pruning --------------------------------------") if args.model_mode == 'c' or args.model_mode =='a': model.prune_by_percentile( ['conv1'], q=100-58.0) model.prune_by_percentile( ['conv2'], q=100-22.0) model.prune_by_percentile( ['conv3'], q=100-34.0) model.prune_by_percentile( ['conv4'], q=100-36.0) model.prune_by_percentile( ['conv5'], q=100-53.0) model.prune_by_percentile( ['conv6'], q=100-24.0) model.prune_by_percentile( ['conv7'], q=100-42.0) model.prune_by_percentile( ['conv8'], q=100-32.0) model.prune_by_percentile( ['conv9'], q=100-27.0) model.prune_by_percentile( ['conv10'], q=100-34.0) model.prune_by_percentile( ['conv11'], q=100-35.0) model.prune_by_percentile( ['conv12'], q=100-29.0) model.prune_by_percentile( ['conv13'], q=100-36.0) if args.model_mode == 'd' or args.model_mode == 'a': model.prune_by_percentile( ['fc1'], q=100-10.0) model.prune_by_percentile( ['fc2'], q=100-10.0) model.prune_by_percentile( ['fc3'], q=100-10.0) print("------------------------- After pruning --------------------------------") util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(args, test_loader, model, criterion) torch.save(model, f"{args.save_dir}/model_pruned.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_pruned.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after pruning\t{accuracy}") util.layer2torch(model, f"{args.save_dir}/{args.out_pruned_folder}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}", weight_list) # Retrain print("------------------------- Prune and Retrain ----------------------------") tok="prune_re" util.initial_train(model, args, train_loader, test_loader, tok, use_cuda=True) print("------------------------- After Retraining -----------------------------") util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(args, test_loader, model, criterion) torch.save(model, f"{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_re_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/mmodel_prune_retrain_{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after prune retrain\t{accuracy}") util.layer2torch(model, f"{args.save_dir}/{args.out_pruned_re_folder}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}", weight_list) return model
def train_and_report(model_name, kernel, warp, ard): dataset_dir = os.path.join(MODEL_DIR, DATASET) try: os.makedirs(dataset_dir) except OSError: print "skipping output folder" for fold in xrange(10): fold_dir = os.path.join(SPLIT_DIR, DATASET, str(fold)) train_data = np.loadtxt(os.path.join(fold_dir, 'train')) test_data = np.loadtxt(os.path.join(fold_dir, 'test')) params_file = None output_dir = os.path.join(dataset_dir, str(fold)) try: os.makedirs(output_dir) except OSError: print "skipping output folder" if ard: iso_dir = output_dir.replace('True', 'False') params_file = os.path.join(iso_dir, 'params') gp = util.train_gp_model(train_data, kernel, warp, ard, params_file) util.save_parameters(gp, os.path.join(output_dir, 'params')) util.save_gradients(gp, os.path.join(output_dir, 'grads')) metrics = util.get_metrics(gp, test_data) util.save_metrics(metrics, os.path.join(output_dir, 'metrics')) util.save_cautious_curves(gp, test_data, os.path.join(output_dir, 'curves')) util.save_predictions(gp, test_data, os.path.join(output_dir, 'preds')) asym_metrics = util.get_asym_metrics(gp, test_data) util.save_asym_metrics(asym_metrics, os.path.join(output_dir, 'asym_metrics')) gc.collect(2) # buggy GPy has allocation cycles...
def initial_process(model): print(model) util.print_model_parameters(model) print( "------------------------- Initial training -------------------------------" ) tok = "initial" criterion = nn.CrossEntropyLoss().cuda() util.initial_train(model, args, train_loader, test_loader, tok, use_cuda=True) accuracy = util.validate(args, test_loader, model, criterion) torch.save(model, f"{args.save_dir}/model_initial_end.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight:{args.save_dir}/{args.out_oldweight_folder}") util.log(f"{args.save_dir}/{args.log}", f"model:{args.save_dir}/model_initial_end.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"initial_accuracy {accuracy}") util.layer2torch(model, f"{args.save_dir}/{args.out_oldweight_folder}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_oldweight_folder}", weight_list) return model
def train_and_report(model_name, kernel, warp, ard, likelihood='gaussian'): dataset_dir = os.path.join(MODEL_DIR, DATASET) try: os.makedirs(dataset_dir) except OSError: print "skipping output folder" for fold in xrange(10): fold_dir = os.path.join(SPLIT_DIR, DATASET, str(fold)) train_data = np.loadtxt(os.path.join(fold_dir, 'train')) test_data = np.loadtxt(os.path.join(fold_dir, 'test')) output_dir = os.path.join(dataset_dir, str(fold)) params_file = None if ard: iso_dir = output_dir.replace('True', 'False') params_file = os.path.join(iso_dir, 'params') gp = util.train_gp_model(train_data, kernel, warp, ard, params_file, likelihood=likelihood) metrics = util.get_metrics(gp, test_data) try: os.makedirs(output_dir) except OSError: print "skipping output folder" util.save_parameters(gp, os.path.join(output_dir, 'params')) util.save_metrics(metrics, os.path.join(output_dir, 'metrics')) #util.save_gradients(gp, os.path.join(output_dir, 'grads')) util.save_cautious_curves(gp, test_data, os.path.join(output_dir, 'curves')) util.save_predictions(gp, test_data, os.path.join(output_dir, 'preds'))
def pruning_process(model): print( "------------------------- Before pruning --------------------------------" ) util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(val_loader, model, args) print( "------------------------- pruning CNN--------------------------------------" ) model.prune_by_percentile(['conv1'], q=100 - 66.0) model.prune_by_percentile(['conv2'], q=100 - 12.0) print( "------------------------------- After prune CNN ----------------------------" ) util.print_nonzeros(model, f"{args.save_dir}/{args.log}") prec1 = util.validate(val_loader, model, args) torch.save(model, f"{args.save_dir}/model_pruned.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_pruned.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"prune acc\t{prec1}") util.layer2torch(f"{args.save_dir}/{args.out_pruned_folder}", model) weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}", weight_list) print( "------------------------- start retrain after prune CNN----------------------------" ) util.initial_train(model, args, train_loader, val_loader, 'prune_re') print( "------------------------- After Retraining -----------------------------" ) util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(val_loader, model, args) torch.save(model, f"{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_re_folder}") util.log( f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"prune and retrain acc\t{accuracy}") util.layer2torch(f"{args.save_dir}/{args.out_pruned_re_folder}", model) weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}", weight_list) return model
def quantize_process(model): #util.print_model_parameters(model) print( '---------------------- Before weight sharing ---------------------------' ) criterion = nn.CrossEntropyLoss().cuda() acc = util.validate(args, test_loader, model, criterion) util.log(f"{args.save_dir}/{args.log}", f"accuracy before weight sharing\t{acc}") # Weight sharing old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing( model, args.model_mode, args.bits) print( '----------------------- After weight sharing ---------------------------' ) acc = util.validate(args, test_loader, model, criterion) torch.save(model, f"{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after weight sharing {args.bits}bits\t{acc}") util.layer2torch(model, f"{args.save_dir}/{args.out_quantized_folder}") util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}", new_weight_list) print( '----------------------- quantize retrain -------------------------------' ) util.quantized_retrain(model, args, quantized_index_list, quantized_center_list, train_loader, use_cuda) #acc = util.test(model, test_loader, use_cuda=True) acc = util.validate(args, test_loader, model, criterion) torch.save( model, f"{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel") util.layer2torch(model, f"{args.save_dir}/{args.out_quantized_re_folder}") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_re_folder}") util.log( f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized_bit{args.bits}_retrain{args.reepochs}.ptmodel" ) util.log(f"{args.save_dir}/{args.log}", f"accuracy retrain after weight sharing\t{acc}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}", weight_list) return model
def quantize_process(model): print('------------------------------- accuracy before weight sharing ----------------------------------') acc = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"accuracy before weight sharing\t{acc}") print('------------------------------- accuacy after weight sharing -------------------------------') tempfc1=torch.index_select(model.fc1.weight, 0, model.invrow1.cuda()) model.fc1.weight=torch.nn.Parameter(torch.index_select(tempfc1, 1, model.invcol1.cuda())) tempfc2=torch.index_select(model.fc2.weight, 0, model.invrow2.cuda()) model.fc2.weight=torch.nn.Parameter(torch.index_select(tempfc2, 1, model.invcol2.cuda())) tempfc3=torch.index_select(model.fc3.weight, 0, model.invrow3.cuda()) model.fc3.weight=torch.nn.Parameter(torch.index_select(tempfc3, 1, model.invcol3.cuda())) old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing(model, args.model_mode, args.bits) temp1=torch.index_select(model.fc1.weight, 0, model.rowp1.cuda()) model.fc1.weight=torch.nn.Parameter(torch.index_select(temp1, 1, model.colp1.cuda())) temp2=torch.index_select(model.fc2.weight, 0, model.rowp2.cuda()) model.fc2.weight=torch.nn.Parameter(torch.index_select(temp2, 1, model.colp2.cuda())) temp3=torch.index_select(model.fc3.weight, 0, model.rowp3.cuda()) model.fc3.weight=torch.nn.Parameter(torch.index_select(temp3, 1, model.colp3.cuda())) acc = util.validate(val_loader, model, args) util.save_checkpoint({ 'state_dict': model.state_dict(), 'best_prec1': acc, }, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('quantized',args.alpha))) util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after weight sharing {args.bits}bits\t{acc}") util.layer2torch(f"{args.save_dir}/{args.out_quantized_folder}" , model) util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}", new_weight_list) print('------------------------------- retraining -------------------------------------------') util.quantized_retrain(model, args, quantized_index_list, quantized_center_list, train_loader, val_loader) acc = util.validate(val_loader, model, args) util.save_checkpoint({ 'state_dict': model.state_dict(), 'best_prec1': acc, }, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('quantized_re',args.alpha))) util.layer2torch(f"{args.save_dir}/{args.out_quantized_re_folder}" , model) util.log(f"{args.save_dir}/{args.log}", f"weight:{args.save_dir}/{args.out_quantized_re_folder}") util.log(f"{args.save_dir}/{args.log}", f"model:{args.save_dir}/model_quantized_bit{args.bits}_retrain{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"acc after qauntize and retrain\t{acc}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}", weight_list) return model
def pruning_process(model): print("------------------------- Before pruning --------------------------------") util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(val_loader, model, args) print("------------------------- pruning CNN--------------------------------------") model.prune_by_percentile( ['conv1'], q=100-58.0) model.prune_by_percentile( ['conv2'], q=100-22.0) model.prune_by_percentile( ['conv3'], q=100-34.0) model.prune_by_percentile( ['conv4'], q=100-36.0) model.prune_by_percentile( ['conv5'], q=100-53.0) model.prune_by_percentile( ['conv6'], q=100-24.0) model.prune_by_percentile( ['conv7'], q=100-42.0) model.prune_by_percentile( ['conv8'], q=100-32.0) model.prune_by_percentile( ['conv9'], q=100-27.0) model.prune_by_percentile( ['conv10'], q=100-34.0) model.prune_by_percentile( ['conv11'], q=100-35.0) model.prune_by_percentile( ['conv12'], q=100-29.0) model.prune_by_percentile( ['conv13'], q=100-36.0) print("------------------------------- After prune CNN ----------------------------") util.print_nonzeros(model, f"{args.save_dir}/{args.log}") prec1 = util.validate(val_loader, model, args) util.save_checkpoint({ 'state_dict': model.state_dict(), 'best_prec1': prec1, }, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('pruned',args.alpha))) util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_pruned.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"prune acc\t{prec1}") util.layer2torch(f"{args.save_dir}/{args.out_pruned_folder}" , model) weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}", weight_list) print("------------------------- start retrain after prune CNN----------------------------") util.initial_train(model, args, train_loader, val_loader, 'prune_re') print("------------------------- After Retraining -----------------------------") util.print_nonzeros(model, f"{args.save_dir}/{args.log}") accuracy = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_re_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"prune and retrain acc\t{accuracy}") util.layer2torch(f"{args.save_dir}/{args.out_pruned_re_folder}" , model) weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}", weight_list) return model
def quantize_process(model): print( '------------------------------- accuracy before weight sharing ----------------------------------' ) acc = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"accuracy before weight sharing\t{acc}") print( '------------------------------- accuacy after weight sharing -------------------------------' ) old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing( model, args.model_mode, args.bits) acc = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after weight sharing {args.bits}bits\t{acc}") util.layer2torch(f"{args.save_dir}/{args.out_quantized_folder}", model) util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}", new_weight_list) print( '------------------------------- retraining -------------------------------------------' ) util.quantized_retrain(model, args, quantized_index_list, quantized_center_list, train_loader, val_loader) acc = util.validate(val_loader, model, args) util.layer2torch(f"{args.save_dir}/{args.out_quantized_re_folder}", model) util.log(f"{args.save_dir}/{args.log}", f"weight:{args.save_dir}/{args.out_quantized_re_folder}") util.log( f"{args.save_dir}/{args.log}", f"model:{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel" ) util.log(f"{args.save_dir}/{args.log}", f"acc after qauntize and retrain\t{acc}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}", weight_list) return model
def initial_process(model): print(model) util.print_model_parameters(model) print("------------------------- Initial training -------------------------------") util.initial_train(model, args, train_loader, val_loader, 'initial') accuracy = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_oldweight_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_initial_end.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"initial_accuracy\t{accuracy}") util.layer2torch(f"{args.save_dir}/{args.out_oldweight_folder}",model) weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_oldweight_folder}", weight_list) return model
def quantize_process(model): print( '------------------------------- accuracy before weight sharing ----------------------------------' ) acc = util.validate(val_loader, model, args) util.log(f"{args.save_dir}/{args.log}", f"accuracy before weight sharing\t{acc}") print( '------------------------------- accuacy after weight sharing -------------------------------' ) tempfc1 = torch.index_select(model.fc1.weight, 0, model.invrow1.cuda()) model.fc1.weight = torch.nn.Parameter( torch.index_select(tempfc1, 1, model.invcol1.cuda())) tempfc2 = torch.index_select(model.fc2.weight, 0, model.invrow2.cuda()) model.fc2.weight = torch.nn.Parameter( torch.index_select(tempfc2, 1, model.invcol2.cuda())) old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing( model, args.model_mode, args.bits) temp1 = torch.index_select(model.fc1.weight, 0, model.rowp1.cuda()) model.fc1.weight = torch.nn.Parameter( torch.index_select(temp1, 1, model.colp1.cuda())) temp2 = torch.index_select(model.fc2.weight, 0, model.rowp2.cuda()) model.fc2.weight = torch.nn.Parameter( torch.index_select(temp2, 1, model.colp2.cuda())) acc = util.validate(val_loader, model, args) torch.save(model, f"{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_folder}") util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized.ptmodel") util.log(f"{args.save_dir}/{args.log}", f"accuracy after weight sharing {args.bits}bits\t{acc}") util.layer2torch(f"{args.save_dir}/{args.out_quantized_folder}", model) util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}", new_weight_list) print( '------------------------------- retraining -------------------------------------------' ) util.quantized_retrain(model, args, quantized_index_list, quantized_center_list, train_loader, val_loader) acc = util.validate(val_loader, model, args) torch.save( model, f"{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel") util.layer2torch(f"{args.save_dir}/{args.out_quantized_re_folder}", model) util.log(f"{args.save_dir}/{args.log}", f"weight:{args.save_dir}/{args.out_quantized_re_folder}") util.log( f"{args.save_dir}/{args.log}", f"model:{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel" ) util.log(f"{args.save_dir}/{args.log}", f"acc after qauntize and retrain\t{acc}") weight_list = util.parameters2list(model.children()) util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}", weight_list) return model