def sensitivity_analysis(model, criterion, data_loader, loggers, args, sparsities): # This sample application can be invoked to execute Sensitivity Analysis on your # model. The ouptut is saved to CSV and PNG. msglogger.info("Running sensitivity tests") if not isinstance(loggers, list): loggers = [loggers] test_fnc = partial( classifier.test, test_loader=data_loader, criterion=criterion, loggers=loggers, args=args, activations_collectors=classifier.create_activation_stats_collectors( model)) which_params = [param_name for param_name, _ in model.named_parameters()] sensitivity = distiller.perform_sensitivity_analysis( model, net_params=which_params, sparsities=sparsities, test_func=test_fnc, group=args.sensitivity) distiller.sensitivities_to_png( sensitivity, os.path.join(msglogger.logdir, 'sensitivity.png')) distiller.sensitivities_to_csv( sensitivity, os.path.join(msglogger.logdir, 'sensitivity.csv'))
def sensitivity_analysis(model, criterion, data_loader, loggers, args): # This sample application can be invoked to execute Sensitivity Analysis on your # model. The ouptut is saved to CSV and PNG. msglogger.info("Running sensitivity tests") if not isinstance(loggers, list): loggers = [loggers] test_fnc = partial(test, test_loader=data_loader, criterion=criterion, loggers=loggers, args=args) which_params = [param_name for param_name, _ in model.named_parameters()] sensitivity = distiller.perform_sensitivity_analysis(model, net_params=which_params, sparsities=np.arange(0.0, 0.95, 0.05), test_func=test_fnc, group=args.sensitivity) distiller.sensitivities_to_png(sensitivity, 'sensitivity.png') distiller.sensitivities_to_csv(sensitivity, 'sensitivity.csv')
def sensitivity_analysis(model, criterion, data_loader, opt, sparsities, msglogger): msglogger.info("Running sensitivity tests") pylogger = PythonLogger(msglogger) if not isinstance(pylogger, list): pylogger = [pylogger] # 可以调用此示例应用程序来对模型执行敏感性分析。输出保存到csv和png。 test_fnc = partial(test, test_loader=data_loader, criterion=criterion, loggers=pylogger, args=opt, msglogger=msglogger, activations_collectors=create_activation_stats_collectors(model)) which_params = [param_name for param_name, _ in model.named_parameters()] sensitivity = distiller.perform_sensitivity_analysis(model, net_params=which_params, sparsities=sparsities, test_func=test_fnc, group=opt.sensitivity) distiller.sensitivities_to_png(sensitivity, 'sensitivity.png') distiller.sensitivities_to_csv(sensitivity, 'sensitivity.csv')
def sensitivity_analysis(args): model = load_state_dict(args) model.eval() model.cuda() sensitivities = np.arange(0.0, 0.95, 0.1) which_params = [param_name for param_name, _ in model.named_parameters()] start_time = time.time() sensitivity = perform_sensitivity_analysis(model, which_params, sensitivities, args) end_time = time.time() print('剪枝敏感度分析总共耗时{}h'.format((end_time - start_time) / 3600)) # distiller.sensitivities_to_png(sensitivity, 'work_space/sensitivity_data/sensitivity_{}.png'.format(args.model)) distiller.sensitivities_to_csv( sensitivity, os.path.join(args.sensitivity_csv_path, 'sensitivity_{}_{}.csv'.format(args.model, get_time())))
def sensitivity_analysis(model, sensitivity_file, test_func, sense_type): """ 分析网络权重的敏感度, 就是剪枝后的performance loss Arguments: model (class Net): 已训练好的模型 \n sensitivity_file (str): 存储敏感度分析结果的文件名称和位置 \n test_func (func): 在Tester中定义的Test Process, 因为要得到performance, 需要每次剪枝后inference一次model \n sense_type (str): 敏感度分析的维度(one of 'element'/'filter'/'channel') Examples: >>> from apputils.platform_summaries import * >>> sensitivity_analysis(model, 'sense_file.xlsx', test_func, 'element') """ msglogger.info("Running sensitivity tests") # test_func = partial(self.test, opt=opt) which_params = [param_name for param_name, _ in model.named_parameters()] sensitivity = distiller.perform_sensitivity_analysis_sr( model, net_params=which_params, sparsities=np.arange(0.0, 0.95, 0.05), test_func=test_func, group=sense_type) distiller.sensitivities_to_csv(sensitivity, sensitivity_file)
def main(): global msglogger check_pytorch_version() args = parser.parse_args() if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) msglogger = apputils.config_pylogger( os.path.join(script_dir, 'logging.conf'), args.name, args.output_dir) # Log various details about the execution environment. It is sometimes useful # to refer to past experiment executions and this information may be useful. apputils.log_execution_env_state(sys.argv, gitroot=module_path) msglogger.debug("Distiller: %s", distiller.__version__) start_epoch = 0 best_top1 = 0 if args.deterministic: # Experiment reproducibility is sometimes important. Pete Warden expounded about this # in his blog: https://petewarden.com/2018/03/19/the-machine-learning-reproducibility-crisis/ # In Pytorch, support for deterministic execution is still a bit clunky. if args.workers > 1: msglogger.error( 'ERROR: Setting --deterministic requires setting --workers/-j to 0 or 1' ) exit(1) # Use a well-known seed, for repeatability of experiments torch.manual_seed(0) random.seed(0) np.random.seed(0) cudnn.deterministic = True else: # This issue: https://github.com/pytorch/pytorch/issues/3659 # Implies that cudnn.benchmark should respect cudnn.deterministic, but empirically we see that # results are not re-produced when benchmark is set. So enabling only if deterministic mode disabled. cudnn.benchmark = True if args.gpus is not None: try: args.gpus = [int(s) for s in args.gpus.split(',')] except ValueError: msglogger.error( 'ERROR: Argument --gpus must be a comma-separated list of integers only' ) exit(1) available_gpus = torch.cuda.device_count() for dev_id in args.gpus: if dev_id >= available_gpus: msglogger.error( 'ERROR: GPU device ID {0} requested, but only {1} devices available' .format(dev_id, available_gpus)) exit(1) # Set default device in case the first one on the list != 0 torch.cuda.set_device(args.gpus[0]) # Infer the dataset from the model name args.dataset = 'cifar10' if 'cifar' in args.arch else 'imagenet' # Create the model png_summary = args.summary is not None and args.summary.startswith('png') is_parallel = not png_summary and args.summary != 'compute' # For PNG summary, parallel graphs are illegible model = create_model(args.pretrained, args.dataset, args.arch, parallel=is_parallel, device_ids=args.gpus) compression_scheduler = None # Create a couple of logging backends. TensorBoardLogger writes log files in a format # that can be read by Google's Tensor Board. PythonLogger writes to the Python logger. tflogger = TensorBoardLogger(msglogger.logdir) pylogger = PythonLogger(msglogger) # We can optionally resume from a checkpoint if args.resume: model, compression_scheduler, start_epoch = apputils.load_checkpoint( model, chkpt_file=args.resume) if 'resnet' in args.arch and 'preact' not in args.arch and 'cifar' in args.arch: distiller.resnet_cifar_remove_layers(model) #model = distiller.resnet_cifar_remove_channels(model, compression_scheduler.zeros_mask_dict) # Define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) msglogger.info('Optimizer Type: %s', type(optimizer)) msglogger.info('Optimizer Args: %s', optimizer.defaults) # This sample application can be invoked to produce various summary reports. if args.summary: which_summary = args.summary if which_summary.startswith('png'): apputils.draw_img_classifier_to_file( model, 'model.png', args.dataset, which_summary == 'png_w_params') else: distiller.model_summary(model, which_summary, args.dataset) exit() # Load the datasets: the dataset to load is inferred from the model name passed # in args.arch. The default dataset is ImageNet, but if args.arch contains the # substring "_cifar", then cifar10 is used. train_loader, val_loader, test_loader, _ = apputils.load_data( args.dataset, os.path.expanduser(args.data), args.batch_size, args.workers, args.validation_size, args.deterministic) msglogger.info('Dataset sizes:\n\ttraining=%d\n\tvalidation=%d\n\ttest=%d', len(train_loader.sampler), len(val_loader.sampler), len(test_loader.sampler)) activations_sparsity = None if args.activation_stats: # If your model has ReLU layers, then those layers have sparse activations. # ActivationSparsityCollector will collect information about this sparsity. # WARNING! Enabling activation sparsity collection will significantly slow down training! activations_sparsity = ActivationSparsityCollector(model) if args.sensitivity is not None: # This sample application can be invoked to execute Sensitivity Analysis on your # model. The ouptut is saved to CSV and PNG. msglogger.info("Running sensitivity tests") test_fnc = partial(test, test_loader=test_loader, criterion=criterion, loggers=[pylogger], print_freq=args.print_freq) which_params = [ param_name for param_name, _ in model.named_parameters() ] sensitivity = distiller.perform_sensitivity_analysis( model, net_params=which_params, sparsities=np.arange(0.0, 0.50, 0.05) if args.sensitivity == 'filter' else np.arange(0.0, 0.95, 0.05), test_func=test_fnc, group=args.sensitivity) distiller.sensitivities_to_png(sensitivity, 'sensitivity.png') distiller.sensitivities_to_csv(sensitivity, 'sensitivity.csv') exit() if args.evaluate: # This sample application can be invoked to evaluate the accuracy of your model on # the test dataset. # You can optionally quantize the model to 8-bit integer before evaluation. # For example: # python3 compress_classifier.py --arch resnet20_cifar ../data.cifar10 -p=50 --resume=checkpoint.pth.tar --evaluate if args.quantize: model.cpu() quantizer = quantization.SymmetricLinearQuantizer(model, 8, 8) quantizer.prepare_model() model.cuda() top1, _, _ = test(test_loader, model, criterion, [pylogger], args.print_freq) if args.quantize: checkpoint_name = 'quantized' apputils.save_checkpoint(0, args.arch, model, optimizer=None, best_top1=top1, name='_'.split(args.name, checkpoint_name) if args.name else checkpoint_name, dir=msglogger.logdir) exit() if args.compress: # The main use-case for this sample application is CNN compression. Compression # requires a compression schedule configuration file in YAML. compression_scheduler = distiller.file_config(model, optimizer, args.compress) for epoch in range(start_epoch, start_epoch + args.epochs): # This is the main training loop. msglogger.info('\n') if compression_scheduler: compression_scheduler.on_epoch_begin(epoch) # Train for one epoch train(train_loader, model, criterion, optimizer, epoch, compression_scheduler, loggers=[tflogger, pylogger], print_freq=args.print_freq, log_params_hist=args.log_params_histograms) distiller.log_weights_sparsity(model, epoch, loggers=[tflogger, pylogger]) if args.activation_stats: distiller.log_activation_sparsity(epoch, loggers=[tflogger, pylogger], collector=activations_sparsity) # evaluate on validation set top1, top5, vloss = validate(val_loader, model, criterion, [pylogger], args.print_freq, epoch) stats = ('Peformance/Validation/', OrderedDict([('Loss', vloss), ('Top1', top1), ('Top5', top5)])) distiller.log_training_progress(stats, None, epoch, steps_completed=0, total_steps=1, log_freq=1, loggers=[tflogger]) if compression_scheduler: compression_scheduler.on_epoch_end(epoch) # remember best top1 and save checkpoint is_best = top1 > best_top1 best_top1 = max(top1, best_top1) apputils.save_checkpoint(epoch, args.arch, model, optimizer, compression_scheduler, best_top1, is_best, args.name, msglogger.logdir) # Finally run results on the test set test(test_loader, model, criterion, [pylogger], args.print_freq)
], [ 'rfb5_1.branch0.0.weight', 'rfb5_1.branch1.0.weight', 'rfb5_1.branch1.1.weight', 'rfb5_1.branch1.2.weight', 'rfb5_1.branch1.3.weight', 'rfb5_1.branch2.0.weight', 'rfb5_1.branch2.1.weight', 'rfb5_1.branch2.2.weight', 'rfb5_1.branch2.3.weight', 'rfb5_1.branch3.0.weight', 'rfb5_1.branch3.1.weight', 'rfb5_1.branch3.2.weight', 'rfb5_1.branch3.3.weight', 'rfb5_1.conv_cat.weight', 'rfb5_1.conv_res.weight' ], [ 'agg1.conv_upsample1.weight', 'agg1.conv_upsample2.weight', 'agg1.conv_upsample3.weight', 'agg1.conv_upsample4.weight', 'agg1.conv_upsample5.weight', 'agg1.conv_concat2.weight', 'agg1.conv_concat3.weight', 'agg1.conv4.weight', 'agg1.conv5.weight' ]] for net_params, module in zip( params, ['darknet', 'rfb3_1', 'rfb4_1', 'rfb5_1', 'agg1']): sensitivity = distiller.perform_sensitivity_analysis(model, net_params=net_params, sparsities=np.arange( 0, 1, 0.05), test_func=test_func, group='filter') distiller.sensitivities_to_csv(sensitivity, 'sensitivity_{}.csv'.format(module)) print('Complete')