def test_policy_scheduling(): model = create_model(False, 'cifar10', 'resnet20_cifar') scheduler = distiller.CompressionScheduler(model) policy = distiller.PruningPolicy(None, None) with pytest.raises(AssertionError): scheduler.add_policy(policy) with pytest.raises(AssertionError): # Test for mutual-exclusive configuration scheduler.add_policy(policy, epochs=[1, 2, 3], starting_epoch=4, ending_epoch=5, frequency=1) scheduler.add_policy(policy, epochs=None, starting_epoch=4, ending_epoch=5, frequency=1) # Regression test for issue #176 - https://github.com/NervanaSystems/distiller/issues/176 scheduler.add_policy(policy, epochs=[1, 2, 3]) sched_metadata = scheduler.sched_metadata[policy] assert sched_metadata['starting_epoch'] == 1 assert sched_metadata['ending_epoch'] == 4 assert sched_metadata['frequency'] is None scheduler.add_policy(policy, epochs=[5]) sched_metadata = scheduler.sched_metadata[policy] assert sched_metadata['starting_epoch'] == 5 assert sched_metadata['ending_epoch'] == 6 assert sched_metadata['frequency'] is None
def dict_config(model, optimizer, sched_dict, scheduler=None): app_cfg_logger.debug('Schedule contents:\n' + json.dumps(sched_dict, indent=2)) if scheduler is None: scheduler = distiller.CompressionScheduler(model) pruners = __factory('pruners', model, sched_dict) regularizers = __factory('regularizers', model, sched_dict) quantizers = __factory('quantizers', model, sched_dict, optimizer=optimizer) if len(quantizers) > 1: raise ValueError("\nError: Multiple Quantizers not supported") extensions = __factory('extensions', model, sched_dict) try: lr_policies = [] for policy_def in sched_dict['policies']: policy = None if 'pruner' in policy_def: try: instance_name, args = __policy_params(policy_def, 'pruner') except TypeError as e: print( '\n\nFatal Error: a policy is defined with a null pruner' ) print( 'Here\'s the policy definition for your reference:\n{}' .format(json.dumps(policy_def, indent=1))) raise assert instance_name in pruners, "Pruner {} was not defined in the list of pruners".format( instance_name) pruner = pruners[instance_name] policy = distiller.PruningPolicy(pruner, args) elif 'regularizer' in policy_def: instance_name, args = __policy_params(policy_def, 'regularizer') assert instance_name in regularizers, "Regularizer {} was not defined in the list of regularizers".format( instance_name) regularizer = regularizers[instance_name] if args is None: policy = distiller.RegularizationPolicy(regularizer) else: policy = distiller.RegularizationPolicy( regularizer, **args) elif 'quantizer' in policy_def: instance_name, args = __policy_params(policy_def, 'quantizer') assert instance_name in quantizers, "Quantizer {} was not defined in the list of quantizers".format( instance_name) quantizer = quantizers[instance_name] policy = distiller.QuantizationPolicy(quantizer) elif 'lr_scheduler' in policy_def: # LR schedulers take an optimizer in their CTOR, so postpone handling until we're certain # a quantization policy was initialized (if exists) lr_policies.append(policy_def) continue elif 'extension' in policy_def: instance_name, args = __policy_params(policy_def, 'extension') assert instance_name in extensions, "Extension {} was not defined in the list of extensions".format( instance_name) extension = extensions[instance_name] policy = extension else: raise ValueError( "\nFATAL Parsing error while parsing the pruning schedule - unknown policy [%s]" .format(policy_def)) add_policy_to_scheduler(policy, policy_def, scheduler) # Any changes to the optmizer caused by a quantizer have occured by now, so safe to create LR schedulers lr_schedulers = __factory('lr_schedulers', model, sched_dict, optimizer=optimizer) for policy_def in lr_policies: instance_name, args = __policy_params(policy_def, 'lr_scheduler') assert instance_name in lr_schedulers, "LR-scheduler {} was not defined in the list of lr-schedulers".format( instance_name) lr_scheduler = lr_schedulers[instance_name] policy = distiller.LRPolicy(lr_scheduler) add_policy_to_scheduler(policy, policy_def, scheduler) except AssertionError: # propagate the assertion information raise except Exception as exception: print("\nFATAL Parsing error!\n%s" % json.dumps(policy_def, indent=1)) print("Exception: %s %s" % (type(exception), exception)) raise return scheduler
def perform_sensitivity_analysis(model, net_params, sparsities, test_func, group): """Perform a sensitivity test for a model's weights parameters. The model should be trained to maximum accuracy, because we aim to understand the behavior of the model's performance in relation to pruning of a specific weights tensor. By default this function will test all of the model's parameters. The return value is a complex sensitivities dictionary: the dictionary's key is the name (string) of the weights tensor. The value is another dictionary, where the tested sparsity-level is the key, and a (top1, top5, loss) tuple is the value. Below is an example of such a dictionary: .. code-block:: python {'features.module.6.weight': {0.0: (56.518, 79.07, 1.9159), 0.05: (56.492, 79.1, 1.9161), 0.10: (56.212, 78.854, 1.9315), 0.15: (35.424, 60.3, 3.0866)}, 'classifier.module.1.weight': {0.0: (56.518, 79.07, 1.9159), 0.05: (56.514, 79.07, 1.9159), 0.10: (56.434, 79.074, 1.9138), 0.15: (54.454, 77.854, 2.3127)} } The test_func is expected to execute the model on a test/validation dataset, and return the results for top1 and top5 accuracies, and the loss value. """ if group not in ['element', 'filter', 'channel']: raise ValueError( "group parameter contains an illegal value: {}".format(group)) sensitivities = OrderedDict() for param_name in net_params: if model.state_dict()[param_name].dim() not in [2, 4]: continue # Make a copy of the model, because when we apply the zeros mask (i.e. # perform pruning), the model's weights are altered model_cpy = deepcopy(model) sensitivity = OrderedDict() for sparsity_level in sparsities: sparsity_level = float(sparsity_level) msglogger.info("Testing sensitivity of %s [%0.1f%% sparsity]" % (param_name, sparsity_level * 100)) # Create the pruner (a level pruner), the pruning policy and the # pruning schedule. if group == 'element': # Element-wise sparasity sparsity_levels = {param_name: sparsity_level} pruner = distiller.pruning.SparsityLevelParameterPruner( name='sensitivity', levels=sparsity_levels) elif group == 'filter': # Filter ranking if model.state_dict()[param_name].dim() != 4: continue regims = {param_name: [sparsity_level, '3D']} pruner = distiller.pruning.L1RankedStructureParameterPruner( name='sensitivity', reg_regims=regims) elif group == 'channel': # Filter ranking if model.state_dict()[param_name].dim() != 4: continue regims = {param_name: [sparsity_level, 'Channels']} pruner = distiller.pruning.L1RankedStructureParameterPruner( name='sensitivity', reg_regims=regims) policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler = CompressionScheduler(model_cpy) scheduler.add_policy(policy, epochs=[0]) # Compute the pruning mask per the pruner and apply the mask on the weights scheduler.on_epoch_begin(0) scheduler.apply_mask() # Test and record the performance of the pruned model prec1, prec5, loss = test_func(model=model_cpy) sensitivity[sparsity_level] = (prec1, prec5, loss) sensitivities[param_name] = sensitivity return sensitivities
def dictConfig(model, optimizer, schedule, sched_dict, logger): logger.debug(json.dumps(sched_dict, indent=1)) pruners = __factory('pruners', model, sched_dict) regularizers = __factory('regularizers', model, sched_dict) lr_schedulers = __factory('lr_schedulers', model, sched_dict, optimizer=optimizer) extensions = __factory('extensions', model, sched_dict) try: for policy_def in sched_dict['policies']: policy = None if 'pruner' in policy_def: try: instance_name, args = __policy_params(policy_def, 'pruner') except TypeError as e: print( '\n\nFatal Error: a policy is defined with a null pruner' ) print( 'Here\'s the policy definition for your reference:\n{}' .format(json.dumps(policy_def, indent=1))) exit(1) assert instance_name in pruners, "Pruner {} was not defined in the list of pruners".format( instance_name) pruner = pruners[instance_name] policy = distiller.PruningPolicy(pruner, args) elif 'regularizer' in policy_def: instance_name, args = __policy_params(policy_def, 'regularizer') assert instance_name in regularizers, "Regularizer {} was not defined in the list of regularizers".format( instance_name) regularizer = regularizers[instance_name] if args is None: policy = distiller.RegularizationPolicy(regularizer) else: policy = distiller.RegularizationPolicy( regularizer, **args) elif 'lr_scheduler' in policy_def: instance_name, args = __policy_params(policy_def, 'lr_scheduler') assert instance_name in lr_schedulers, "LR-scheduler {} was not defined in the list of lr-schedulers".format( instance_name) lr_scheduler = lr_schedulers[instance_name] policy = distiller.LRPolicy(lr_scheduler) elif 'extension' in policy_def: instance_name, args = __policy_params(policy_def, 'extension') assert instance_name in extensions, "Extension {} was not defined in the list of extensions".format( instance_name) extension = extensions[instance_name] policy = extension else: print( "\nFATAL Parsing error while parsing the pruning schedule - unknown policy [%s]" % policy_def) exit(1) if 'epochs' in policy_def: schedule.add_policy(policy, epochs=policy_def['epochs']) else: schedule.add_policy( policy, starting_epoch=policy_def['starting_epoch'], ending_epoch=policy_def['ending_epoch'], frequency=policy_def['frequency']) except AssertionError: # propagate the assertion information raise except Exception as exception: print("\nFATAL Parsing error!\n%s" % json.dumps(policy_def, indent=1)) print("Exception: %s %s" % (type(exception), exception)) exit(1) return schedule
def objective(space): global model global count global global_min_score #Explore new model model = create_model(False, args.dataset, args.arch, device_ids=args.gpus) count += 1 # Objective function: F(Acc, Lat) = (1 - Acc.) + (alpha * Sparsity) accuracy = 0 alpha = 0.3 # Super-parameter: the importance of inference time latency = 0.0 sparsity = 0.0 # Training hyperparameter if args.resume: model, compression_scheduler, start_epoch = apputils.load_checkpoint( model, chkpt_file=args.resume) print('resume mode: {}'.format(args.resume)) print(global_min_score) criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) """ distiller/distiller/config.py # Element-wise sparsity sparsity_levels = {net_param: sparsity_level} pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels) policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler = distiller.CompressionScheduler(model) scheduler.add_policy(policy, epochs=[0, 2, 4]) # Local search add multiple pruner for each layer """ sparsity_levels = {} for key, value in space.items(): sparsity_levels[key] = value #print(sparsity_levels) pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels) # for SparsityLevelParameterPruner # pruner = distiller.pruning.SensitivityPruner(name='sensitivity', sensitivities=sparsity_levels) # for SensitivityPruner policy = distiller.PruningPolicy(pruner, pruner_args=None) lrpolicy = distiller.LRPolicy(torch.optim.lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.1)) compression_scheduler = distiller.CompressionScheduler(model) compression_scheduler.add_policy(policy, epochs=[PrunerEpoch]) # compression_scheduler.add_policy(policy, starting_epoch=0, ending_epoch=38, frequency=2) compression_scheduler.add_policy(lrpolicy, starting_epoch=0, ending_epoch=50, frequency=1) """ distiller/example/classifier_compression/compress_classifier.py For each epoch: compression_scheduler.on_epoch_begin(epoch) train() save_checkpoint() compression_scheduler.on_epoch_end(epoch) train(): For each training step: compression_scheduler.on_minibatch_begin(epoch) output = model(input) loss = criterion(output, target) compression_scheduler.before_backward_pass(epoch) loss.backward() optimizer.step() compression_scheduler.on_minibatch_end(epoch) """ local_min_score = 2. for i in range(args.epochs): compression_scheduler.on_epoch_begin(i) train_accuracy = train(i,criterion, optimizer, compression_scheduler) val_accuracy = validate() # Validate hyperparameter setting t, sparsity = distiller.weights_sparsity_tbl_summary(model, return_total_sparsity=True) compression_scheduler.on_epoch_end(i, optimizer) apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, False, 'hyperopt', './') print('Epoch: {}, train_acc: {:.4f}, val_acc: {:.4f}, sparsity: {:.4f}'.format(i, train_accuracy, val_accuracy, sparsity)) score = (1-(val_accuracy/100.)) + (alpha * (1-sparsity/100.)) # objective funtion here if(score < global_min_score): global_min_score = score apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, True, 'best', './') if(score < local_min_score): local_min_score = score if (PrunerConstraint == True and i >= PrunerEpoch and (sparsity < Expected_Sparsity_Level_Low or sparsity > Expected_Sparsity_Level_High)): break test_accuracy = test() # Validate hyperparameter setting print('{} trials: score: {:.4f}, train_acc:{:.4f}, val_acc:{:.4f}, test_acc:{:.4f}, sparsity:{:.4f}'.format(count, local_min_score, train_accuracy, val_accuracy, test_accuracy, sparsity)) return local_min_score
def objective(space): global model global count global best_dict #Explore new model model = create_model(False, args.dataset, args.arch, device_ids=args.gpus) if args.resume: model, _, _ = apputils.load_checkpoint( model, chkpt_file=args.resume) count += 1 print('{} trial starting...'.format(count)) # Objective function: F(Acc, Lat) = (1 - Acc.) + (alpha * Sparsity) accuracy = 0 #alpha = 0.2 # Super-parameter: the importance of inference time alpha = 1.0 # Super-parameter: the importance of inference time sparsity = 0.0 # Training hyperparameter criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) """ distiller/distiller/config.py # Element-wise sparsity sparsity_levels = {net_param: sparsity_level} pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels) policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler = distiller.CompressionScheduler(model) scheduler.add_policy(policy, epochs=[0, 2, 4]) # Local search add multiple pruner for each layer """ sparsity_levels = {} for key, value in space.items(): sparsity_levels[key] = value pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels) policy = distiller.PruningPolicy(pruner, pruner_args=None) lrpolicy = distiller.LRPolicy(torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)) compression_scheduler = distiller.CompressionScheduler(model) #compression_scheduler.add_policy(policy, epochs=[90]) compression_scheduler.add_policy(policy, epochs=[0]) compression_scheduler.add_policy(lrpolicy, starting_epoch=0, ending_epoch=90, frequency=1) """ distiller/example/classifier_compression/compress_classifier.py For each epoch: compression_scheduler.on_epoch_begin(epoch) train() save_checkpoint() compression_scheduler.on_epoch_end(epoch) train(): For each training step: compression_scheduler.on_minibatch_begin(epoch) output = model(input) loss = criterion(output, target) compression_scheduler.before_backward_pass(epoch) loss.backward() optimizer.step() compression_scheduler.on_minibatch_end(epoch) """ for i in range(args.epochs): compression_scheduler.on_epoch_begin(i) train_accuracy = train(i,criterion, optimizer, compression_scheduler) val_accuracy = validate() # Validate hyperparameter setting t, sparsity = distiller.weights_sparsity_tbl_summary(model, return_total_sparsity=True) compression_scheduler.on_epoch_end(i, optimizer) apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, False, 'hyperopt', './') print('{} epochs => train acc:{:.2f}%, val acc:{:.2f}%'.format(i, train_accuracy, val_accuracy)) test_accuracy = validate(test_loader) # Validate hyperparameter setting #score = (1-(val_accuracy/100.)) + (alpha * (1-sparsity/100.)) # objective funtion here # objective funtion here # accuracy: 98~90%, sparsity: 80%~50% score = -((val_accuracy/100.)**2-0.9**2 + alpha * ((sparsity/100.)**2-0.5**2)) print('{} trials: score: {:.2f}\ttrain acc:{:.2f}%\tval acc:{:.2f}%\ttest acc:{:.2f}%\tsparsity:{:.2f}%'.format(count, score, train_accuracy, val_accuracy, test_accuracy, sparsity)) if score < best_dict['score']: best_dict['trial'] = count best_dict['score'] = score best_dict['tr_acc'] = train_accuracy best_dict['v_acc'] = val_accuracy best_dict['te_acc'] = test_accuracy best_dict['sparsity'] = sparsity best_dict['model_best'] = copy.deepcopy(model) return score
def perform_sensitivity_analysis(model, net_params, sparsities, args): sensitivities = OrderedDict() print('测试原模型精度') accuracy = test(args, model) print('原模型精度为:{}'.format(accuracy)) if args.fpgm: print('即将采用几何中位数剪枝产生折线图') conv_dict = {} if args.hrank: print('即将采用HRank剪枝') cnt = 1 layer_name = 'conv1.conv.weight' conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 for key, value in model.block_info.items(): if value == 1: layer_name = '{}.conv.conv.weight'.format(key) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = '{}.conv_dw.conv.weight'.format(key) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = '{}.project.conv.weight'.format(key) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 else: for j in range(value): layer_name = '{}.model.{}.conv.conv.weight'.format(key, j) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = '{}.model.{}.conv_dw.conv.weight'.format( key, j) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = '{}.model.{}.project.conv.weight'.format( key, j) conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = 'conv_6_sep.conv.weight' conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 layer_name = 'conv_6_dw.conv.weight' conv_dict[layer_name] = np.load(args.rank_path + 'rank_conv' + str(cnt) + '.npy') cnt += 1 print(len(conv_dict)) for param_name in net_params: if model.state_dict()[param_name].dim() not in [4]: continue model_cpy = deepcopy(model) sensitivity = OrderedDict() # 对每一层循环剪枝并测试精度(从0.05->0.95) for sparsity_level in sparsities: sparsity_level = float(sparsity_level) print(param_name, sparsity_level) pruner = distiller.pruning.L1RankedStructureParameterPruner( "sensitivity", group_type="Filters", desired_sparsity=sparsity_level, weights=param_name) policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler = CompressionScheduler(model_cpy) scheduler.add_policy(policy, epochs=[0]) scheduler.on_epoch_begin(0, fpgm=args.fpgm, HRank=args.hrank, conv_index=conv_dict) scheduler.mask_all_weights() accuracy = test(args, model_cpy) print('剪枝{}后的精度为:{}'.format(sparsity_level, accuracy)) sensitivity[sparsity_level] = (accuracy, 0, 0) sensitivities[param_name] = sensitivity return sensitivities
initial_sparsity=0.05, final_sparsity=0.20, weights=['module.conv2.weight']) gemm_pruner_agp = distiller.AutomatedGradualPruner( name='gemm_pruner_agp', initial_sparsity=0.02, final_sparsity=0.15, weights=['module.fc2.weight']) net_thinner = distiller.FilterRemover("remove_filters", arch='simplenet_mnist', dataset='mnist') # Policies policy1 = distiller.PruningPolicy(filter_pruner, pruner_args=None) compression_scheduler.add_policy(policy1, epochs=(0, 1)) policy2 = distiller.PruningPolicy(filter_pruner_agp, pruner_args=None) compression_scheduler.add_policy(policy2, starting_epoch=0, ending_epoch=2, frequency=1) policy3 = distiller.PruningPolicy(gemm_pruner_agp, pruner_args=None) compression_scheduler.add_policy(policy3, starting_epoch=0, ending_epoch=2, frequency=1) compression_scheduler.add_policy(net_thinner, epochs=(2, ))
def perform_model_pruning(model, sparsity_level, group): all_params = [param_name for param_name, param in model.named_parameters()] pruner_list = [] first_layer_name = all_params[0] last_layer_name = all_params[-2] if group == 'element': sparsity_level = float(sparsity_level) # Element-wise sparasity # This can be well deployed in every layer of the deep neural network. sparsity_level = {} for param_name in all_params: if model.state_dict()[param_name].dim() not in [2,4]: continue sparsity_levels = {param_name: sparsity_level[param_name]} pruner = distiller.pruning.SparsityLevelParameterPruner(name="sensitivity", levels=sparsity_levels) pruner_list.append(pruner) elif group == 'filter': # One thing you should keep in mind, in this case, the output layer will be executed by channel-wise analysis rather than filter. for param_name in all_params: if model.state_dict()[param_name].dim() not in [2,4]: continue #sparsity_levels = {param_name: sparsity_level[param_name]} desired_sparsity = sparsity_level[param_name] if not desired_sparsity: continue if param_name == last_layer_name: pruner = L1RankedStructureParameterPruner("sensitivity", group_type="Channels", desired_sparsity=desired_sparsity, weights=last_layer_name) else: pruner = L1RankedStructureParameterPruner("sensitivity", group_type="Filters", desired_sparsity=desired_sparsity, weights=param_name) pruner_list.append(pruner) elif group == 'channel': for param_name in all_params: if param_name == first_layer_name: continue if model.state_dict()[param_name].dim() not in [2,4] or param_name == first_layer_name: continue desired_sparsity = sparsity_level[param_name] if not desired_sparsity: continue pruner = distiller.pruning.L1RankedStructureParameterPruner("sensitivity", group_type="Channels", desired_sparsity=desired_sparsity, weights=param_name) pruner_list.append(pruner) # Build scheduler to set zero mask dictionary. scheduler = CompressionScheduler(model) for pruner in pruner_list: policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler.add_policy(policy, epochs=[0]) # Compute the pruning mask per the pruner and apply the mask on the weights scheduler.on_epoch_begin(0) scheduler.mask_all_weights(epoch=0) # Build fake optimizer, the reason we call if fake is that there are no training loops here. optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4) criterion = nn.CrossEntropyLoss().to('cpu') model.to(device) input_shape = (1, 3, 32, 32) #print(compress_scheduler.zeros_mask_dict['basic_model.fc2.weight'].mask) dummy_input = utility.get_dummy_input('cifar10', # Dataset should be specified. utility.model_device(model), input_shape=input_shape) sgraph = SummaryGraph(model, dummy_input) if group == 'filter': # First remove filter thinning_recipe = thinning.create_thinning_recipe_filters(sgraph, model, scheduler.zeros_mask_dict, prune_output_layer=None) thinning.apply_and_save_recipe(model, scheduler.zeros_mask_dict, thinning_recipe, optimizer) # Second remove channel from last layer. zeros_mask_dict = create_model_masks_dict(model) zeros_mask_dict[last_layer_name].mask = scheduler.zeros_mask_dict[last_layer_name].mask #print(zeros_mask_dict[last_layer_name].mask) thinning_recipe = thinning.create_thinning_recipe_channels(sgraph, model, zeros_mask_dict) thinning.apply_and_save_recipe(model, scheduler.zeros_mask_dict, thinning_recipe, optimizer) elif group == 'channel': thinning_recipe = thinning.create_thinning_recipe_channels(sgraph, model, zeros_mask_dict) thinning.apply_and_save_recipe(model, scheduler.zeros_mask_dict, thinning_recipe, optimizer) else: raise ValueError("Can not execute thinning recipe with the model pruned by element-wise mode.") return model
def perform_sensitivity_analysis(model, net_params, sparsities, test_func, group): """Perform a sensitivity test for a model's weights parameters. The model should be trained to maximum accuracy, because we aim to understand the behavior of the model's performance in relation to pruning of a specific weights tensor. By default this function will test all of the model's parameters. The return value is a complex sensitivities dictionary: the dictionary's key is the name (string) of the weights tensor. The value is another dictionary, where the tested sparsity-level is the key, and a (top1, top5, loss) tuple is the value. Below is an example of such a dictionary: .. code-block:: python {'features.module.6.weight': {0.0: (56.518, 79.07, 1.9159), 0.05: (56.492, 79.1, 1.9161), 0.10: (56.212, 78.854, 1.9315), 0.15: (35.424, 60.3, 3.0866)}, 'classifier.module.1.weight': {0.0: (56.518, 79.07, 1.9159), 0.05: (56.514, 79.07, 1.9159), 0.10: (56.434, 79.074, 1.9138), 0.15: (54.454, 77.854, 2.3127)} } The test_func is expected to execute the model on a test/validation dataset, and return the results for top1 and top5 accuracies, and the loss value. """ if group not in ['element', 'filter', 'channel']: raise ValueError( "group parameter contains an illegal value: {}".format(group)) sensitivities = OrderedDict() # Terminated layer for channel pruning last_layer_name = net_params[-2][0] first_layer_name = net_params[0][0] for param_name, param_var in net_params: # Ignore bias analysis. if model.state_dict()[param_name].dim() not in [2, 4]: continue # Make a copy of the model, because when we apply the zeros mask (i.e. # perform pruning), the model's weights are altered model_cpy = deepcopy(model) layer_wise_eval_scores_dict = {} sensitivity = OrderedDict() for sparsity_level in sparsities: sparsity_level = float(sparsity_level) msglogger.info("Testing sensitivity of %s [%0.1f%% sparsity]" % (param_name, sparsity_level * 100)) # ************** # Should I specify the pruner or not? Qualcomm directly use the SVD pruner to get the optimial permutation of filter, and the sensitivity table could also be optimal? # ************** # Create the pruner (a level pruner) # pruning schedule. if group == 'element': # Element-wise sparasity # This can be well deployed in every layer of the deep neural network. sparsity_levels = {param_name: sparsity_level} pruner = distiller.pruning.SparsityLevelParameterPruner( name="sensitivity", levels=sparsity_levels) elif group == 'filter': # Output filter ranking # But so far it's only restricted to convolution layers. # ********************* # My idea: (How about adding new ) # The last layer (i.e output layer) can not be examined and pruned in this case, distiller's authors thus droped the analysis process to avoid trivial code writing. # How to define the termination condition? and apply this mechanism to other fully connected layers. I think this is really important, since there is too many weights here. # ********************* if param_name == last_layer_name: # One thing you should keep in mind, in this case, the output layer will be executed by channel-wise analysis rather than filter. pruner = L1RankedStructureParameterPruner( "sensitivity", group_type="Channels", desired_sparsity=sparsity_level, weights=param_name) else: pruner = L1RankedStructureParameterPruner( "sensitivity", group_type="Filters", desired_sparsity=sparsity_level, weights=param_name) """ if model.state_dict()[param_name].dim() != 4: continue pruner = distiller.pruning.L1RankedStructureParameterPruner("sensitivity", group_type="Filters", desired_sparsity=sparsity_level, weights=param_name) """ elif group == 'channel': # Input channel ranking # But so far it's only restricted to convolution layers. # ********************* # My idea: # Channel pruning is very applicable and more suitable to deploy (no change the size of output logit), but distiiler not support analyze, either. # The reason is that pruner should be specified the group type by YAML file, so they decide dropping this code to avoid trivial code gerneration. # *Should I add new sensitivity analysis policy or pruner*? # ********************* """ if model.state_dict()[param_name].dim() != 4: continue """ if param_name == first_layer_name: continue pruner = distiller.pruning.L1RankedStructureParameterPruner( "sensitivity", group_type="Channels", desired_sparsity=sparsity_level, weights=param_name) policy = distiller.PruningPolicy(pruner, pruner_args=None) scheduler = CompressionScheduler(model_cpy) scheduler.add_policy(policy, epochs=[0]) # Compute the pruning mask per the pruner and apply the mask on the weights scheduler.on_epoch_begin(0) scheduler.mask_all_weights(epoch=0) # Test and record the performance of the pruned model record = test_func(model=model_cpy, parameter_name=param_name) layer_wise_eval_scores_dict[1 - sparsity_level] = record[0] sensitivity[sparsity_level] = record + (param_var, ) sensitivities[param_name] = sensitivity #Our testing function output is consisted of nat_top1, nat_top5, nat_loss, adv_top1, adv_top5, adv_loss. eval_scores_dict[param_name] = layer_wise_eval_scores_dict return sensitivities, eval_scores_dict