def dump_csv(predictor, true_output): results = [] n_predictors = len(predictor.index2combine_name) path_json_dumps = [ '%s/%s/result_%s_%s.json' % (args.root_path, predictor.index2combine_name[i], predictor.index2policy[i], args.flag) for i in range(n_predictors) ] print('Start eval predictor...') predictions = predictor.fusion_prediction(top=1, return_with_prob=True) if len(results) == 0: for i in range(len(predictions)): results.append([]) for index, prediction in enumerate(predictions): results[index] = prediction assert len(results) == len( path_json_dumps ), 'The result length is not equal with path_json_dumps\'s.' for result, save_path in zip(results, path_json_dumps): dir = os.path.dirname(save_path) if not os.path.exists(dir): os.makedirs(dir) if args.flag == 'valid': class_2_index = {0: 'normal', 1: 'phone', 2: 'smoke'} dets_info = {} pred_idx = [] with open("./txts/v-info-new.json", 'r', encoding="utf-8") as f: shape_dict = json.load(f) for i, res in enumerate(result): name = dataset.filenames()[i].split('/')[-1].split('.')[0] dets_info[name] = [ class_2_index[res[0][0]], float(res[0][1]), shape_dict[name][1], shape_dict[name][2] ] pred_idx.append(res[0][0]) with open(save_path, "w", encoding="utf-8") as f: json.dump(dets_info, f) acc = accuracy_score(y_true=true_output, y_pred=pred_idx) test_map = eval_map(detFolder=save_path, gtFolder="txts/v-info-new.json") print("acc score: %.4f, map score: %.4f" % (acc, test_map)) else: class_2_index = {0: 'normal', 1: 'calling', 2: 'smoking'} result_list = [] with open(save_path, 'w', encoding="utf-8") as out_file: filenames = dataset.filenames() for i, res in enumerate(result): filename = filenames[i].split('/')[-1].strip() name = class_2_index[res[0][0]] result_data = { "image_name": str(filename), "category": name, "score": float(res[0][1]) } result_list.append(result_data) json.dump(result_list, out_file) print('Dump %s finished.' % save_path)
def classifier_pred(classifier, shape_path, feature, id, model_name, data_type="valid"): class_2_index = {0: 'normal', 1: 'phone', 2: 'smoke'} dets_info = {} features = pickle.load(open(feature, 'rb')) ids = pickle.load(open(id, 'rb')) predict = classifier.predict(features) # predicted_test_scores = classifier.decision_function(features) # probs = softmax(predicted_test_scores) # prob_list = [prob[int(predict[i])] for i, prob in enumerate(probs)] probs = classifier.predict_proba(features) prob_list = [round(prob[int(predict[i])], 4) for i, prob in enumerate(probs)] prediction = predict.tolist() total_pred_idx = [int(pred) for pred in prediction] total_true_idx = [int(label) for label in ids] with open("./txts/%s.json" % shape_path, 'r', encoding="utf-8") as f: shape_dict = json.load(f) dataset = Dataset(os.path.join(BASE, data_type)) filenames = dataset.filenames() for i, filename in enumerate(filenames): name = filename.split('/')[-1].split('.')[0] dets_info[name] = [class_2_index[int(prediction[i])], prob_list[i], shape_dict[name][1], shape_dict[name][2]] with open("%s/%s.json" % (feature_path, shape_path.split('-')[0]), "w", encoding="utf-8") as f: json.dump(dets_info, f) accuracy = round(accuracy_score(total_true_idx, total_pred_idx), 4) test_map, ap_list = eval_map(detFolder="%s/v.json" % feature_path, gtFolder="txts/v-info-new.json", return_each_ap=True) print("Accuracy: %s, map: %.4f" % (accuracy, test_map)) with open("weights/%s-valid.json" % model_name, 'w', encoding="utf-8") as f: prob_dict = {} prob_dict["prob"] = probs prob_dict["model_weight"] = test_map prob_dict["label_weight"] = ap_list json.dump(prob_dict, f, cls=MyEncoder) return accuracy, round(test_map, 4)
def validate(args): # might as well try to validate something args.prefetcher = not args.no_prefetcher # create model model = create_model( args.model, num_classes=args.num_classes, in_chans=3, global_pool=args.gp) if args.checkpoint: load_checkpoint(model, args.checkpoint) param_count = sum([m.numel() for m in model.parameters()]) _logger.info('Model %s created, param count: %d' % (args.model, param_count)) data_config = resolve_data_config(vars(args), model=model) # model, test_time_pool = apply_test_time_pool(model, data_config, args) if torch.cuda.is_available(): model.cuda() criterion = nn.CrossEntropyLoss().cuda() dataset = Dataset(args.data) crop_pct = data_config['crop_pct'] loader = create_loader( dataset, input_size=data_config['input_size'], batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=data_config['interpolation'], mean=data_config['mean'], std=data_config['std'], num_workers=args.workers, crop_pct=crop_pct) batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() f1_m = AverageMeter() end = time.time() total_pred_idx = [] total_truth_idx = [] mistake_image = [] mistake_image_dict = {'calling': [], 'normal': [], 'smoking': [], 'smoking_calling': []} # class_2_index = {0: 'normal', 1: 'phone', 2: 'smoke'} class_2_index = {0: 'calling', 1: 'normal', 2: 'smoking', 3: 'smoking_calling'} with open("./txts/%s.json" % json_name, 'r', encoding="utf-8") as f: shape_dict = json.load(f) dets_info = {} model.eval() with torch.no_grad(): # warmup, reduce variability of first batch time, especially for comparing torchscript vs non input = torch.randn((args.batch_size,) + data_config['input_size']) if torch.cuda.is_available(): input = input.cuda() model(input) end = time.time() for batch_idx, (input, target) in enumerate(loader): if args.no_prefetcher and torch.cuda.is_available(): target = target.cuda() input = input.cuda() # compute output # t0 = time.time() output = model(input) # print("time0: %.8f s" % ((time.time() - t0))) # t1 = time.time() # out = output.detach().cpu() # print("time1: %.8f s" % ((time.time() - t1) / 64)) # print("time2: %.8f s" % ((time.time() - t0) / 64)) # t2 = time.time() # out = out.cuda().cpu() # print("time3: %.8f s" % ((time.time() - t2) / 64)) # get prediction index and ground turth index prob = torch.max(F.softmax(output, -1), -1)[0] idx = torch.max(F.softmax(output, -1), -1)[1] target_idx = target.cpu().numpy() predict_idx = idx.cpu().numpy() for j in range(len(target_idx)): total_truth_idx.append(target_idx[j]) total_pred_idx.append(predict_idx[j]) class_dict = loader.dataset.class_to_idx target_class = list(class_dict.keys())[list(class_dict.values()).index(int(target_idx[j]))] pred_class = list(class_dict.keys())[list(class_dict.values()).index(int(predict_idx[j]))] filename = loader.dataset.filenames()[batch_idx * args.batch_size + j] name = filename.split('/')[-1].split('.')[0] dets_info[name] = [pred_class, float(prob[j]), shape_dict[name][1], shape_dict[name][2]] if target_idx[j] != predict_idx[j]: mistake_image.append( [loader.dataset.filenames()[batch_idx * args.batch_size + j], target_class, pred_class, np.round(prob[j].cpu().numpy(), 4)]) mistake_image_dict[class_2_index[predict_idx[j]]].append( loader.dataset.filenames()[batch_idx * args.batch_size + j]) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 3)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: _logger.info( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' 'Loss: {loss.val:>7.4f} ({loss.avg:>6.4f}) ' 'Acc@1: {top1.val:>7.2f} ({top1.avg:>7.2f}) ' 'Acc@5: {top5.val:>7.3f} ({top5.avg:>7.3f})'.format( batch_idx, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, loss=losses, top1=top1, top5=top5)) with open("%s/%s.json" % (output_path, json_name.split('-')[0]), "w", encoding="utf-8") as f: json.dump(dets_info, f) top1a, top5a = top1.avg, top5.avg results = OrderedDict( top1=round(top1a, 4), top1_err=round(100 - top1a, 4), top5=round(top5a, 4), top5_err=round(100 - top5a, 4), param_count=round(param_count / 1e6, 2), img_size=data_config['input_size'][-1], cropt_pct=crop_pct, interpolation=data_config['interpolation'], mistake_image_dict=mistake_image_dict, pred_idx=total_pred_idx, truth_idx=total_truth_idx) _logger.info(' * Acc@1 {:.2f} ({:.2f}) Acc@5 {:.2f} ({:.2f})'.format( results['top1'], results['top1_err'], results['top5'], results['top5_err'])) map, each_ap = eval_map(detFolder="%s/%s.json" % (output_path, json_name.split('-')[0]), gtFolder="txts/%s.json" % json_name, return_each_ap=True) _logger.info('Valid mAP: {}, each ap: {}'.format(round(map, 4), each_ap)) return results
def main(): setup_default_logging() args, args_text = _parse_args() args.prefetcher = not args.no_prefetcher torch.manual_seed(args.seed) model = create_model(args.model, pretrained=True, num_classes=args.num_classes, drop_rate=args.drop, drop_path_rate=args.drop_path, drop_block_rate=args.drop_block, checkpoint_path=args.initial_checkpoint) if args.local_rank == 0: _logger.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) data_config = resolve_data_config(vars(args), model=model, verbose=args.local_rank == 0) if args.num_gpu > 1: model = nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda() else: model.cuda() optimizer = create_optimizer(args, model) loss_scaler = None # optionally resume from a checkpoint resume_epoch = None if args.resume: resume_epoch = resume_checkpoint( model, args.resume, optimizer=None if args.no_resume_opt else optimizer, loss_scaler=None if args.no_resume_opt else loss_scaler, log_info=args.local_rank == 0) lr_scheduler, num_epochs = create_scheduler(args, optimizer) start_epoch = 0 if args.start_epoch is not None: # a specified start_epoch will always override the resume epoch start_epoch = args.start_epoch elif resume_epoch is not None: start_epoch = resume_epoch if lr_scheduler is not None and start_epoch > 0: lr_scheduler.step(start_epoch) if args.local_rank == 0: _logger.info('Scheduled epochs: {}'.format(num_epochs)) train_dir = os.path.join(args.data, 'train') if not os.path.exists(train_dir): _logger.error( 'Training folder does not exist at: {}'.format(train_dir)) exit(1) dataset_train = Dataset(train_dir) collate_fn = None mixup_fn = None mixup_active = args.mixup > 0 or args.cutmix > 0. or args.cutmix_minmax is not None if mixup_active: mixup_args = dict(mixup_alpha=args.mixup, cutmix_alpha=args.cutmix, cutmix_minmax=args.cutmix_minmax, prob=args.mixup_prob, switch_prob=args.mixup_switch_prob, elementwise=args.mixup_elem, label_smoothing=args.smoothing, num_classes=args.num_classes) if args.prefetcher: collate_fn = FastCollateMixup(**mixup_args) else: mixup_fn = Mixup(**mixup_args) loader_train = create_loader( dataset_train, input_size=data_config['input_size'], batch_size=args.batch_size, is_training=True, use_prefetcher=args.prefetcher, re_prob=args.reprob, re_mode=args.remode, re_count=args.recount, color_jitter=args.color_jitter, auto_augment=args.aa, num_workers=args.workers, collate_fn=collate_fn, ) eval_dir = os.path.join(args.data, 'valid') if not os.path.isdir(eval_dir): eval_dir = os.path.join(args.data, 'validation') if not os.path.isdir(eval_dir): _logger.error( 'Validation folder does not exist at: {}'.format(eval_dir)) exit(1) dataset_eval = Dataset(eval_dir) loader_eval = create_loader( dataset_eval, input_size=data_config['input_size'], batch_size=args.batch_size, is_training=False, use_prefetcher=args.prefetcher, num_workers=args.workers, crop_pct=data_config['crop_pct'], ) if mixup_active: # smoothing is handled with mixup target transform train_loss_fn = SoftTargetCrossEntropy().cuda() elif args.smoothing: train_loss_fn = LabelSmoothingCrossEntropy( smoothing=args.smoothing).cuda() train_loss_ce = nn.CrossEntropyLoss().cuda() else: train_loss_fn = nn.CrossEntropyLoss().cuda() validate_loss_fn = nn.CrossEntropyLoss().cuda() eval_metric = args.eval_metric best_metric = None best_epoch = None saver = None output_dir = '' plateau_num = 0 if args.local_rank == 0: output_base = args.output if args.output else './output' exp_name = '-'.join([ datetime.now().strftime("%Y%m%d-%H%M%S"), args.model, str(data_config['input_size'][-1]) ]) output_dir = get_outdir(output_base, exp_name) decreasing = True if eval_metric == 'loss' else False saver = CheckpointSaver(model=model, optimizer=optimizer, args=args, amp_scaler=loss_scaler, checkpoint_dir=output_dir, recovery_dir=output_dir, decreasing=decreasing, max_history=2) with open(os.path.join(output_dir, 'args.yaml'), 'w') as f: f.write(args_text) with open("./txts/%s.json" % json_name, 'r', encoding="utf-8") as f: shape_dict = json.load(f) try: for epoch in range(start_epoch, num_epochs): train_metrics = train_epoch(epoch, model, loader_train, optimizer, [train_loss_fn, train_loss_ce], args, lr_scheduler=lr_scheduler, output_dir=output_dir, mixup_fn=mixup_fn) eval_metrics, dets_info = validate(model, loader_eval, validate_loss_fn, args, shape_dict=shape_dict) with open("%s/v.json" % output_dir, "w", encoding="utf-8") as f: json.dump(dets_info, f) map = round( eval_map(detFolder="%s/v.json" % output_dir, gtFolder="txts/%s.json" % json_name), 4) eval_metrics["map"] = map _logger.info('Valid mAP: {}'.format(map)) if lr_scheduler is not None: # step LR for next epoch lr_scheduler.step(epoch + 1, eval_metrics[eval_metric]) update_summary(epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'), write_header=best_metric is None) if saver is not None: # save proper checkpoint with eval metric save_metric = eval_metrics[eval_metric] saver.save_prefix = "%.2f-%s" % (eval_metrics["top1"], map) best_metric, best_epoch = saver.save_checkpoint( epoch, metric=save_metric) if eval_metrics[eval_metric] >= best_metric: plateau_num = 0 else: plateau_num += 1 # 超过30个epoch还没有更新metric,停止运行 if plateau_num == 30: break except KeyboardInterrupt: pass if best_metric is not None: _logger.info('*** Best metric: {0} (epoch {1})'.format( best_metric, best_epoch))
best_map = 0 for i in range(150): save_path = feature_path + '%s.m' % classifier fier = classifier_training(train_features, classifier, train_label_path) acc, test_map = classifier_pred(fier, "v-info-new", valid_features, valid_label_path, model_name) # test_map = eval_map(detFolder="%s/v.json" % feature_path, gtFolder="txts/v-info-new.json") # print(classifier, test_map) test_map = round(test_map, 4) if test_map > best_map: if i != 0: last_path = save_path.replace(classifier, classifier + "-%s" % best_map) os.remove(last_path) best_map = test_map save_path = save_path.replace(classifier, classifier + "-%s" % test_map) joblib.dump(fier, save_path) elif stage == "valid": save_path = glob.glob('%s%s*.m' % (feature_path, classifier))[0] fier = joblib.load(save_path) classifier_pred(fier, "v-info-new", valid_feature_path, valid_label_path, save_name) elif stage == "eval": valid_model(model_name, checkpoints) test_map = eval_map(detFolder="%s/v.json" % feature_path, gtFolder="txts/v-info-new.json") print(test_map) # save_feature_batch(model_name, checkpoints, train_feature_path, train_label_path, "valid") else: save_path = glob.glob('%s%s*.m' % (feature_path, classifier))[0] classifier_test(save_path, test_feature_path, test_imgs)
def perform_predict(predictor, loader, model_weight, label_weight, weights, save_weights=True): temp_weight = {} total_true_output = [] total_pred_output = [] total_pred_idx = [] total_true_idx = [] right_count = 0 n_labels = np.zeros((3, )) + 1e-5 n_right_labels = np.zeros((3, )) with torch.no_grad(): for i, (input, target) in enumerate(tqdm(loader)): test_pred_tta = [] for j in range(8): output = predictor(input.cuda()) output = output.data.cpu().numpy() test_pred_tta.append(output) output_data = softmax(np.mean(test_pred_tta, axis=0)) # output = predictor(input.cuda()) # output_data = test_pred_tta.cpu().numpy() if save_weights: predict_idx = np.argmax(output, axis=-1) target_idx = target.cpu().numpy() total_pred_idx.extend(predict_idx) total_true_idx.extend(target_idx) for j in range(len(target_idx)): # 统计预测中预测对的数量,相当于precision n_labels[predict_idx[j]] += 1 # 统计真实中预测对的数量,相当于recall # n_labels[target_idx[j]] += 1 if predict_idx[j] == target_idx[j]: right_count += 1 n_right_labels[predict_idx[j]] += 1 total_true_output.append(target_idx[j]) total_pred_output.append(output_data[j]) else: total_pred_output.extend(output_data) model_name = model.default_cfg['model_name'].split('-')[1] if save_weights: # model_weight[predictor.default_cfg['model_name']] = np.array([float(right_count) / len(total_true_output)]) # label_weight[predictor.default_cfg['model_name']] = n_right_labels / n_labels # # temp_weight['model_weight'] = float(right_count) / len(total_true_output) # temp_weight['label_weight'] = list(n_right_labels / n_labels) # weights[predictor.default_cfg['model_name']] = temp_weight # # with open('./weights/fusion_weights_tta.json', 'w') as json_file: # json.dump(weights, json_file, indent=4) with open("./txts/v-info-new.json", 'r', encoding="utf-8") as f: shape_dict = json.load(f) filenames = dataset.filenames() dets_info = {} class_2_index = {0: 'normal', 1: 'phone', 2: 'smoke'} probs = np.max(softmax(np.array(total_pred_output)), axis=-1) for i, filename in enumerate(filenames): name = filename.split('/')[-1].split('.')[0] dets_info[name] = [ class_2_index[int(total_pred_idx[i])], probs[i], shape_dict[name][1], shape_dict[name][2] ] with open("fusions/fv.json", "w", encoding="utf-8") as f: json.dump(dets_info, f, cls=MyEncoder) accuracy = round(accuracy_score(total_true_idx, total_pred_idx), 4) test_map, ap_list = eval_map(detFolder="fusions/fv.json", gtFolder="txts/v-info-new.json", return_each_ap=True) print("Accuracy: %s, map: %.4f" % (accuracy, test_map)) with open("weights/fusion_weights_map.json", 'w', encoding="utf-8") as f: weights[predictor.default_cfg['model_name']] = {} weights[ predictor.default_cfg['model_name']]["model_weight"] = test_map weights[ predictor.default_cfg['model_name']]["label_weight"] = ap_list json.dump(weights, f, cls=MyEncoder, indent=2) model_weight[predictor.default_cfg['model_name']] = test_map label_weight[predictor.default_cfg['model_name']] = ap_list else: with open('./weights/fusion_weights_tta.json', 'r') as json_file: json_data = json.load(json_file) model_weight[predictor.default_cfg['model_name']] = np.array( [json_data[predictor.default_cfg['model_name']]['model_weight']]) label_weight[predictor.default_cfg['model_name']] = np.array( [json_data[predictor.default_cfg['model_name']]['label_weight']]) return total_pred_output, total_true_output