if quantize_lambda is not None: if isinstance(modelz, nn.DataParallel): modelz.module.replace_bias(model.module, weight_name) else: modelz.replace_bias(model, weight_name) acc, acc_adv = model_test(modelz, epoch, test_loader, atk_algo=attack_algo, atk_eps=args.attack_eps, iscuda=args.cuda, adv_iter=args.defend_iter, criterion=F.cross_entropy) layers = layers_unique(modelz, normalized=False, param_name=weight_name)[1] if quantize_lambda is not None: dist, dist_layer, relative_layer = util_trts.model_distance( model, modelz, weight_name=weight_name) print("\t model dist: {}, layer-wise dist :{}, relative: {}". format(dist, dist_layer, relative_layer)) if dist < best_dist: best_dist = dist misc.model_saver(model, args.savedir, args.save_model_name, "sparse_closest_" + descripter) misc.model_saver(modelz, args.savedir, args.save_model_name, "quant_closest_" + descripter) print("\t\t closest model saved")
if isinstance(quant_model, nn.DataParallel): quant_model.module.replace_bias(sparse_model.module, weight_name) else: quant_model.replace_bias(sparse_model, weight_name) acc, acc_adv = model_test(quant_model, 0, test_loader, atk_algo=attack_algo, atk_eps=args.attack_eps, iscuda=args.cuda, adv_iter=args.attack_iter, criterion=F.cross_entropy) layers = layers_nnz(quant_model, param_name=weight_name)[1] misc.print_dict(layers, name="NNZ PER LAYER") # print(all_num) sparse_factor = sum(layers.values()) # print(sparse_factor) if quantize_algo is not None: if args.quantize_algo == "kmeans_nnz_fixed_0_center": weight_bits = [math.ceil(math.log2(item-1)) if item > 1 else 0 for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] weight_dict_size = [ (item - 1) * 32 for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] elif args.quantize_algo == "kmeans": weight_bits = [math.ceil(math.log2(item)) for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] weight_dict_size = [item * 32 for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] elif args.quantize_algo == "kmeans_fixed_nnz": weight_bits = [math.ceil(math.log2(item-1)) if item > 1 else 0 for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] weight_dict_size = [ (item - 1) * 32 for item in list(pt.layers_unique(quant_model, weight_name)[1].values())] else: raise 'quantize algo error!' else: weight_bits = [32] * len(layers) model_size = 0 for i in range(len(layers)): dict_size = weight_dict_size[i] if quantize_algo is not None else 0 nnz_cur = list(layers.values())[i]
atk_algo=attack_algo, atk_eps=args.attack_eps, iscuda=args.cuda, adv_iter=args.attack_iter, criterion=F.cross_entropy) layers = layers_nnz(model, param_name=weight_name)[1] # misc.print_dict(layers, name="NNZ PER LAYER") # print(all_num) sparse_factor = sum(layers.values()) # print(sparse_factor) if quantize_algo is not None: if args.quantize_algo == "kmeans_nnz_fixed_0_center": weight_bits = [ math.ceil(math.log2(item - 1)) if item > 1 else 0 for item in list( pt.layers_unique(model, weight_name)[1].values()) ] weight_dict_size = [(item - 1) * 32 for item in list( pt.layers_unique(model, weight_name)[1].values())] elif args.quantize_algo == "kmeans": weight_bits = [ math.ceil(math.log2(item)) for item in list( pt.layers_unique(model, weight_name)[1].values()) ] weight_dict_size = [ item * 32 for item in list( pt.layers_unique(model, weight_name)[1].values()) ] elif args.quantize_algo == "kmeans_fixed_nnz": weight_bits = [ math.ceil(math.log2(item - 1)) if item > 1 else 0