def test(net, val_data, use_cuda, calc_weight_count=False, calc_flops=False, extended_log=False): acc_top1 = AverageMeter() acc_top5 = AverageMeter() tic = time.time() err_top1_val, err_top5_val = validate( acc_top1=acc_top1, acc_top5=acc_top5, net=net, val_data=val_data, use_cuda=use_cuda) if calc_weight_count: weight_count = calc_net_weight_count(net) logging.info('Model: {} trainable parameters'.format(weight_count)) if calc_flops: n_flops, n_params = measure_model(net, 224, 224) logging.info('Params: {} ({:.2f}M), FLOPs: {} ({:.2f}M)'.format( n_params, n_params / 1e6, n_flops, n_flops / 1e6)) if extended_log: logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format( top1=err_top1_val, top5=err_top5_val)) else: logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format( top1=err_top1_val, top5=err_top5_val)) logging.info('Time cost: {:.4f} sec'.format( time.time() - tic))
def test(net, val_data, use_cuda, input_image_size, in_channels, calc_weight_count=False, calc_flops=False, calc_flops_only=True, extended_log=False): if not calc_flops_only: acc_top1 = AverageMeter() acc_top5 = AverageMeter() tic = time.time() err_top1_val, err_top5_val = validate(acc_top1=acc_top1, acc_top5=acc_top5, net=net, val_data=val_data, use_cuda=use_cuda) if extended_log: logging.info( 'Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})' .format(top1=err_top1_val, top5=err_top5_val)) else: logging.info( 'Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format( top1=err_top1_val, top5=err_top5_val)) logging.info('Time cost: {:.4f} sec'.format(time.time() - tic)) if calc_weight_count: weight_count = calc_net_weight_count(net) if not calc_flops: logging.info('Model: {} trainable parameters'.format(weight_count)) if calc_flops: num_flops, num_macs, num_params = measure_model( net, in_channels, input_image_size) assert (not calc_weight_count) or (weight_count == num_params) stat_msg = "Params: {params} ({params_m:.2f}M), FLOPs: {flops} ({flops_m:.2f}M)," \ " FLOPs/2: {flops2} ({flops2_m:.2f}M), MACs: {macs} ({macs_m:.2f}M)" logging.info( stat_msg.format(params=num_params, params_m=num_params / 1e6, flops=num_flops, flops_m=num_flops / 1e6, flops2=num_flops / 2, flops2_m=num_flops / 2 / 1e6, macs=num_macs, macs_m=num_macs / 1e6))
def test(net, test_data, metric, use_cuda, input_image_size, in_channels, calc_weight_count=False, calc_flops=False, calc_flops_only=True, extended_log=False): if not calc_flops_only: tic = time.time() validate( metric=metric, net=net, val_data=test_data, use_cuda=use_cuda) accuracy_msg = report_accuracy( metric=metric, extended_log=extended_log) logging.info("Test: {}".format(accuracy_msg)) logging.info("Time cost: {:.4f} sec".format( time.time() - tic)) if calc_weight_count: weight_count = calc_net_weight_count(net) if not calc_flops: logging.info("Model: {} trainable parameters".format(weight_count)) if calc_flops: num_flops, num_macs, num_params = measure_model(net, in_channels, input_image_size) assert (not calc_weight_count) or (weight_count == num_params) stat_msg = "Params: {params} ({params_m:.2f}M), FLOPs: {flops} ({flops_m:.2f}M)," \ " FLOPs/2: {flops2} ({flops2_m:.2f}M), MACs: {macs} ({macs_m:.2f}M)" logging.info(stat_msg.format( params=num_params, params_m=num_params / 1e6, flops=num_flops, flops_m=num_flops / 1e6, flops2=num_flops / 2, flops2_m=num_flops / 2 / 1e6, macs=num_macs, macs_m=num_macs / 1e6))
def test(net, test_data, use_cuda, input_image_size, in_channels, num_classes, calc_weight_count=False, calc_flops=False, calc_flops_only=True, extended_log=False, dataset_metainfo=None): assert (dataset_metainfo is not None) if not calc_flops_only: metric = [] pix_acc_macro_average = False metric.append( PixelAccuracyMetric(vague_idx=dataset_metainfo["vague_idx"], use_vague=dataset_metainfo["use_vague"], macro_average=pix_acc_macro_average)) mean_iou_macro_average = False metric.append( MeanIoUMetric(num_classes=num_classes, vague_idx=dataset_metainfo["vague_idx"], use_vague=dataset_metainfo["use_vague"], bg_idx=dataset_metainfo["background_idx"], ignore_bg=dataset_metainfo["ignore_bg"], macro_average=mean_iou_macro_average)) tic = time.time() accuracy_info = validate1(accuracy_metrics=metric, net=net, val_data=test_data, use_cuda=use_cuda) pix_acc = accuracy_info[0][1] mean_iou = accuracy_info[1][1] pix_macro = "macro" if pix_acc_macro_average else "micro" iou_macro = "macro" if mean_iou_macro_average else "micro" if extended_log: logging.info( "Test: {pix_macro}-pix_acc={pix_acc:.4f} ({pix_acc}), " "{iou_macro}-mean_iou={mean_iou:.4f} ({mean_iou})".format( pix_macro=pix_macro, pix_acc=pix_acc, iou_macro=iou_macro, mean_iou=mean_iou)) else: logging.info( "Test: {pix_macro}-pix_acc={pix_acc:.4f}, {iou_macro}-mean_iou={mean_iou:.4f}" .format(pix_macro=pix_macro, pix_acc=pix_acc, iou_macro=iou_macro, mean_iou=mean_iou)) logging.info('Time cost: {:.4f} sec'.format(time.time() - tic)) if calc_weight_count: weight_count = calc_net_weight_count(net) if not calc_flops: logging.info('Model: {} trainable parameters'.format(weight_count)) if calc_flops: num_flops, num_macs, num_params = measure_model( net, in_channels, input_image_size) assert (not calc_weight_count) or (weight_count == num_params) stat_msg = "Params: {params} ({params_m:.2f}M), FLOPs: {flops} ({flops_m:.2f}M)," \ " FLOPs/2: {flops2} ({flops2_m:.2f}M), MACs: {macs} ({macs_m:.2f}M)" logging.info( stat_msg.format(params=num_params, params_m=num_params / 1e6, flops=num_flops, flops_m=num_flops / 1e6, flops2=num_flops / 2, flops2_m=num_flops / 2 / 1e6, macs=num_macs, macs_m=num_macs / 1e6))
def test(net, test_data, metric, use_cuda, input_image_size, in_channels, calc_weight_count=False, calc_flops=False, calc_flops_only=True, extended_log=False, show_bad_samples=False): """ Main test routine. Parameters: ---------- net : Module Model. test_data : DataLoader Data loader. metric : EvalMetric Metric object instance. use_cuda : bool Whether to use CUDA. input_image_size : tuple of 2 ints Spatial size of the expected input image. in_channels : int Number of input channels. calc_weight_count : bool, default False Whether to calculate count of weights. calc_flops : bool, default False Whether to calculate FLOPs. calc_flops_only : bool, default True Whether to only calculate FLOPs without testing. extended_log : bool, default False Whether to log more precise accuracy values. show_bad_samples : bool, default False Whether to log file names for bad samples. """ if not calc_flops_only: tic = time.time() validate( metric=metric, net=net, val_data=test_data, use_cuda=use_cuda) accuracy_msg = report_accuracy( metric=metric, extended_log=extended_log) logging.info("Test: {}".format(accuracy_msg)) logging.info("Time cost: {:.4f} sec".format( time.time() - tic)) if calc_weight_count: weight_count = calc_net_weight_count(net) if not calc_flops: logging.info("Model: {} trainable parameters".format(weight_count)) if calc_flops: num_flops, num_macs, num_params = measure_model(net, in_channels, input_image_size) assert (not calc_weight_count) or (weight_count == num_params) stat_msg = "Params: {params} ({params_m:.2f}M), FLOPs: {flops} ({flops_m:.2f}M)," \ " FLOPs/2: {flops2} ({flops2_m:.2f}M), MACs: {macs} ({macs_m:.2f}M)" logging.info(stat_msg.format( params=num_params, params_m=num_params / 1e6, flops=num_flops, flops_m=num_flops / 1e6, flops2=num_flops / 2, flops2_m=num_flops / 2 / 1e6, macs=num_macs, macs_m=num_macs / 1e6)) if show_bad_samples: store_misses = StoreMisses() validate( metric=store_misses, net=net, val_data=test_data, use_cuda=use_cuda) _, misses_list = store_misses.get() if len(misses_list) > 0: dataset = test_data.iterable.dataset if isinstance(test_data, tqdm) else test_data.dataset for i, miss_ind in enumerate(misses_list): logging.info("Miss [{}]: {}".format(i, dataset.get_file_name(miss_ind)))
def calc_model_accuracy(net, test_data, metric, use_cuda, input_image_size, in_channels, calc_weight_count=False, calc_flops=False, calc_flops_only=True, extended_log=False, ml_type="cls"): """ Estimating particular model accuracy. Parameters: ---------- net : Module Model. test_data : DataLoader Data loader. metric : EvalMetric Metric object instance. use_cuda : bool Whether to use CUDA. input_image_size : tuple of 2 ints Spatial size of the expected input image. in_channels : int Number of input channels. calc_weight_count : bool, default False Whether to calculate count of weights. calc_flops : bool, default False Whether to calculate FLOPs. calc_flops_only : bool, default True Whether to only calculate FLOPs without testing. extended_log : bool, default False Whether to log more precise accuracy values. ml_type : str, default 'cls' Machine learning type. Returns: ------- list of floats Accuracy values. """ if not calc_flops_only: tic = time.time() validate( metric=metric, net=net, val_data=test_data, use_cuda=use_cuda) accuracy_msg = report_accuracy( metric=metric, extended_log=extended_log) logging.info("Test: {}".format(accuracy_msg)) logging.info("Time cost: {:.4f} sec".format( time.time() - tic)) acc_values = metric.get()[1] acc_values = acc_values if type(acc_values) == list else [acc_values] else: acc_values = [] if calc_weight_count: weight_count = calc_net_weight_count(net) if not calc_flops: logging.info("Model: {} trainable parameters".format(weight_count)) if calc_flops: in_shapes = [(1, 640 * 25 * 5), (1,)] if ml_type == "asr" else\ [(1, in_channels, input_image_size[0], input_image_size[1])] num_flops, num_macs, num_params = measure_model( model=net, in_shapes=in_shapes) assert (not calc_weight_count) or (weight_count == num_params) stat_msg = "Params: {params} ({params_m:.2f}M), FLOPs: {flops} ({flops_m:.2f}M)," \ " FLOPs/2: {flops2} ({flops2_m:.2f}M), MACs: {macs} ({macs_m:.2f}M)" logging.info(stat_msg.format( params=num_params, params_m=num_params / 1e6, flops=num_flops, flops_m=num_flops / 1e6, flops2=num_flops / 2, flops2_m=num_flops / 2 / 1e6, macs=num_macs, macs_m=num_macs / 1e6)) return acc_values