def evaluate(model, path, iou_thres, conf_thres, nms_thres, img_size, batch_size): model.eval() # Get dataloader dataset = ListDataset(path, img_size=img_size, augment=False, multiscale=False) dataloader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=1, collate_fn=dataset.collate_fn ) Tensor = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) for batch_i, (_, imgs, targets) in enumerate(tqdm.tqdm(dataloader, desc="Detecting objects")): # Extract labels labels += targets[:, 1].tolist() # Rescale target targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size imgs = Variable(imgs.type(Tensor), requires_grad=False) with torch.no_grad(): outputs = model(imgs) outputs = non_max_suppression(outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thres) # Concatenate sample statistics true_positives, pred_scores, pred_labels = [np.concatenate(x, 0) for x in list(zip(*sample_metrics))] precision, recall, AP, f1, ap_class = ap_per_class(true_positives, pred_scores, pred_labels, labels) return precision, recall, AP, f1, ap_class
def evaluate(model, path, iou_thres, conf_thres, nms_thres, img_size, batch_size): model.eval() # Get dataloader dataset = ListDataset(path, img_size=img_size, augment=False, multiscale=False) dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0, collate_fn=dataset.collate_fn) Tensor = torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) for batch_i, (_, _, imgs, targets) in enumerate( tqdm.tqdm(dataloader, desc="Detecting objects")): # Extract labels if targets is None: continue labels += targets[:, 1].tolist() # Rescale target to x1y1x2y2. YOLO outputs in XYWH and it gets converted in the IOU script later targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size imgs = Variable(imgs.type(Tensor), requires_grad=False) with torch.no_grad(): outputs = model(imgs) outputs = non_max_suppression(outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thres) # Concatenate sample statistics try: true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] precision, recall, AP, f1, ap_class = ap_per_class( true_positives, pred_scores, pred_labels, labels) except ValueError as error: print('-----------------------------------------------') print(error) print('Model failed to detect any boxes in validation above threshold') print('Zeros passed for all metrics') print('-----------------------------------------------') precision, recall, f1 = (None, None, None) AP = np.array([0] * len(np.unique(labels))) ap_class = np.unique(labels).astype("int32") return precision, recall, AP, f1, ap_class
def val(epoch, args, model, val_dataloader, iou_thresh, conf_thresh, nms_thresh, img_size, batch_size=8): global best_mAP print("begin to val the datasets...") model.eval() Tensor = torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor labels = [] sample_metrics = [] for batch_i, (_, imgs, targets) in enumerate( tqdm(val_dataloader, desc="detection the objections:")): labels += targets[:, 1].tolist() targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size imgs = Variable(imgs.type(Tensor), requires_grad=False) with torch.no_grad(): outputs = model(imgs) outputs = non_max_suppression(outputs, conf_thres=conf_thresh, nms_thres=nms_thresh) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thresh) tp, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] precision, recall, AP, f1, ap_class = ap_per_class(tp, pred_scores, pred_labels, labels) val_precision = precision.mean() val_recall = recall.mean() val_f1 = f1.mean() val_mAP = AP.mean() print("precision: %.3f, recall: %.3f, f1: %.3f, mAP: %.3f" % (val_precision, val_recall, val_f1, val_mAP)) if val_mAP > best_mAP: best_mAP = val_mAP save_name = os.path.join(args.save_dir, "best_model_%.6f.pth" % best_mAP) state_dict = model.state_dict() for key in state_dict.keys(): state_dict[key] = state_dict[key].cpu() torch.save({"model": state_dict, "epoch": epoch + 1}, save_name) print("model has been saved in %s" % save_name, end="") return precision, recall, AP, f1, ap_class
def validation_epoch_end(self): stats = [np.concatenate(x, 0) for x in zip(*self.stats)] # to numpy p, r, ap, f1, ap_class = ap_per_class(*stats) if self.niou > 1: p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]] mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=self.nc) # number of targets per class maps = np.zeros(self.nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] # Return results return (mp, mr, map, mf1), maps
def evaluate(model, path, iou_thres, conf_thres, nms_thres, img_size, batch_size, device): model.eval() # Get dataloader dataset = ListDataset(path, img_size=img_size, augment=False, multiscale=False) dataloader = tc.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=1, collate_fn=dataset.collate_fn) labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) try: tq = tqdm.tqdm(dataloader, desc='Detecting objects', ncols=100) for _, imgs, targets in tq: imgs = imgs.to(device) # Extract labels labels += targets[:, 1].tolist() # Rescale target targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size with tc.no_grad(): outputs = model(imgs) outputs = non_max_suppression(outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thres) finally: tq.close() # Concatenate sample statistics true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] precision, recall, AP, f1, ap_class = ap_per_class(true_positives, pred_scores, pred_labels, labels) return precision, recall, AP, f1, ap_class
def evaluate(args, model, model_cfg, test_loader, iou_thres=0.5, conf_thres=0.5, nms_thres=0.5): img_size = int(model_cfg[0]['width']) model.eval() Tensor = torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor labels, sample_metrics = [], [] for batch_i, (_, imgs, targets) in enumerate( tqdm.tqdm(test_loader, desc="Detecting objects")): #if batch_i >10:break labels += targets[:, 1].tolist() targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size #import pdb;pdb.set_trace() imgs = Variable(imgs.type(Tensor), requires_grad=False) with torch.no_grad(): #import pdb;pdb.set_trace() outputs = model(imgs) outputs = non_max_suppression(outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thres) true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] precision, recall, AP, f1, ap_class = ap_per_class(true_positives, pred_scores, pred_labels, labels) return precision, recall, AP, f1, ap_class
def evaluate(output_dir, data_dir, annotation_dir, size, overlap=0.5, filter_str1="", filter_str2=""): """Evaluate output for the ball data.""" files = os.listdir(data_dir) output_files = os.listdir(output_dir) output_files = [x for x in output_files if x.endswith(".txt")] output_imgs = [os.path.splitext(x)[0] for x in output_files] annotations = get_scaled_annotations_PVOC(annotation_dir, size) aps = [] for f in files: annotation = annotations[f] if filter_str1 not in f or filter_str2 not in f: continue if f not in output_imgs: aps.append(0) # print(f, f not in output_imgs) else: preds, conf = read_output_file(os.path.join( output_dir, f + ".txt")) if len(preds) == 0: aps.append(0) else: target_cls = np.zeros(annotation.shape[0]) pred_cls = np.zeros(preds.shape[0]) tps = match_annotations(preds, annotation, overlap) p, r, ap, f1, _ = utils.ap_per_class(tps, conf, pred_cls, target_cls) aps.append(ap[0]) if len(aps) == 0: return 0 mean_ap = sum(aps) / len(aps) return mean_ap
def on_batch_end(self, last_output, last_target, **kwargs): bs = last_output[0].shape[0] iou_thres = torch.tensor((0.5, )) niou = iou_thres.numel() for batch_idx in range(0, bs): target_boxes = last_target[0][batch_idx].cpu() target_classes = last_target[1][batch_idx].cpu() - 1.0 people_idxs = (torch.LongTensor( (0, )) == target_classes).nonzero().view(-1) target_boxes = target_boxes[people_idxs] target_classes = target_classes[people_idxs] yolo_out = grab_idx(last_output, batch_idx) pred = YoloCategoryList.yolo2pred( yolo_out) # list([[x1, y1, x2, y2, conf, cls]]) detections = pred[0] if detections is None: # bs=1, first and only result if len(target_classes): self.stats.append((torch.zeros(0, 1), torch.Tensor(), torch.Tensor(), target_classes)) continue boxes = YoloCategoryList.bbox2fai(detections) correct = torch.zeros(len(detections), niou) if len(target_classes): for det_idx, det in enumerate( detections): # detections per image # Break if all targets already located in image pbox = boxes[det_idx] iou, j = bbox_iou(pbox, target_boxes).max(0) correct[det_idx] = iou > iou_thres conf = detections[:, 4] clazz = detections[:, 5] self.stats.append((correct, conf, clazz, target_classes)) stats = [np.concatenate(x, 0) for x in list(zip(*self.stats))] # to numpy p, r, ap, f1, ap_class = ap_per_class(*stats) self.apAt50 = ap.item()
def test_det( opt, batch_size=12, img_size=(1088, 608), iou_thres=0.5, print_interval=40, ): data_cfg = opt.data_cfg f = open(data_cfg) data_cfg_dict = json.load(f) f.close() nC = 1 test_path = data_cfg_dict['test'] dataset_root = data_cfg_dict['root'] if opt.gpus[0] >= 0: opt.device = torch.device('cuda') else: opt.device = torch.device('cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) model = load_model(model, opt.load_model) #model = torch.nn.DataParallel(model) model = model.to(opt.device) model.eval() # Get dataloader transforms = T.Compose([T.ToTensor()]) dataset = DetDataset(dataset_root, test_path, img_size, augment=False, transforms=transforms) dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=8, drop_last=False, collate_fn=collate_fn) mean_mAP, mean_R, mean_P, seen = 0.0, 0.0, 0.0, 0 print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) outputs, mAPs, mR, mP, TP, confidence, pred_class, target_class, jdict = \ [], [], [], [], [], [], [], [], [] AP_accum, AP_accum_count = np.zeros(nC), np.zeros(nC) for batch_i, (imgs, targets, paths, shapes, targets_len) in enumerate(dataloader): t = time.time() #seen += batch_size output = model(imgs.cuda())[-1] origin_shape = shapes[0] width = origin_shape[1] height = origin_shape[0] inp_height = img_size[1] inp_width = img_size[0] c = np.array([width / 2., height / 2.], dtype=np.float32) s = max(float(inp_width) / float(inp_height) * height, width) * 1.0 meta = { 'c': c, 's': s, 'out_height': inp_height // opt.down_ratio, 'out_width': inp_width // opt.down_ratio } hm = output['hm'].sigmoid_() wh = output['wh'] reg = output['reg'] if opt.reg_offset else None opt.K = 200 detections, inds = mot_decode(hm, wh, reg=reg, cat_spec_wh=opt.cat_spec_wh, K=opt.K) # Compute average precision for each sample targets = [targets[i][:int(l)] for i, l in enumerate(targets_len)] for si, labels in enumerate(targets): seen += 1 #path = paths[si] #img0 = cv2.imread(path) dets = detections[si] dets = dets.unsqueeze(0) dets = post_process(opt, dets, meta) dets = merge_outputs(opt, [dets])[1] #remain_inds = dets[:, 4] > opt.det_thres #dets = dets[remain_inds] if dets is None: # If there are labels but no detections mark as zero AP if labels.size(0) != 0: mAPs.append(0), mR.append(0), mP.append(0) continue # If no labels add number of detections as incorrect correct = [] if labels.size(0) == 0: # correct.extend([0 for _ in range(len(detections))]) mAPs.append(0), mR.append(0), mP.append(0) continue else: target_cls = labels[:, 0] # Extract target boxes as (x1, y1, x2, y2) target_boxes = xywh2xyxy(labels[:, 2:6]) target_boxes[:, 0] *= width target_boxes[:, 2] *= width target_boxes[:, 1] *= height target_boxes[:, 3] *= height ''' path = paths[si] img0 = cv2.imread(path) img1 = cv2.imread(path) for t in range(len(target_boxes)): x1 = target_boxes[t, 0] y1 = target_boxes[t, 1] x2 = target_boxes[t, 2] y2 = target_boxes[t, 3] cv2.rectangle(img0, (x1, y1), (x2, y2), (0, 255, 0), 4) cv2.imwrite('gt.jpg', img0) for t in range(len(dets)): x1 = dets[t, 0] y1 = dets[t, 1] x2 = dets[t, 2] y2 = dets[t, 3] cv2.rectangle(img1, (x1, y1), (x2, y2), (0, 255, 0), 4) cv2.imwrite('pred.jpg', img1) abc = ace ''' detected = [] for *pred_bbox, conf in dets: obj_pred = 0 pred_bbox = torch.FloatTensor(pred_bbox).view(1, -1) # Compute iou with target boxes iou = bbox_iou(pred_bbox, target_boxes, x1y1x2y2=True)[0] # Extract index of largest overlap best_i = np.argmax(iou) # If overlap exceeds threshold and classification is correct mark as correct if iou[best_i] > iou_thres and obj_pred == labels[ best_i, 0] and best_i not in detected: correct.append(1) detected.append(best_i) else: correct.append(0) # Compute Average Precision (AP) per class AP, AP_class, R, P = ap_per_class( tp=correct, conf=dets[:, 4], pred_cls=np.zeros_like(dets[:, 4]), # detections[:, 6] target_cls=target_cls) # Accumulate AP per class AP_accum_count += np.bincount(AP_class, minlength=nC) AP_accum += np.bincount(AP_class, minlength=nC, weights=AP) # Compute mean AP across all classes in this image, and append to image list mAPs.append(AP.mean()) mR.append(R.mean()) mP.append(P.mean()) # Means of all images mean_mAP = np.sum(mAPs) / (AP_accum_count + 1E-16) mean_R = np.sum(mR) / (AP_accum_count + 1E-16) mean_P = np.sum(mP) / (AP_accum_count + 1E-16) if batch_i % print_interval == 0: # Print image mAP and running mean mAP print(('%11s%11s' + '%11.3g' * 4 + 's') % (seen, dataloader.dataset.nF, mean_P, mean_R, mean_mAP, time.time() - t)) # Print mAP per class print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) print('AP: %-.4f\n\n' % (AP_accum[0] / (AP_accum_count[0] + 1E-16))) # Return mAP return mean_mAP, mean_R, mean_P
def test( model, dataloader, iou_thres=0.5, conf_thres=0.3, nms_thres=0.45, print_interval=40, ): nC = 1 mean_mAP, mean_R, mean_P, seen = 0.0, 0.0, 0.0, 0 print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) outputs, mAPs, mR, mP, TP, confidence, pred_class, target_class, jdict = \ [], [], [], [], [], [], [], [], [] AP_accum, AP_accum_count = np.zeros(nC), np.zeros(nC) for batch_i, (imgs, targets, paths, shapes, targets_len) in enumerate(dataloader): t = time.time() out = model(imgs.cuda()) # out = model(imgs) output = [] for i,o in enumerate(out): boxes = xyxy2xywh(o['boxes']).cpu() scores = o['scores'].cpu().view(-1,1) labels = o['labels'].cpu().view(-1,1).float() output.append(torch.Tensor(torch.cat((boxes,scores,scores,labels),dim=1))) output = non_max_suppression(output, conf_thres=conf_thres, nms_thres=nms_thres) for i, o in enumerate(output): if o is not None: output[i] = o[:, :6] # Compute average precision for each sample targets = [targets[i][:int(l)] for i,l in enumerate(targets_len)] for si, (labels, detections) in enumerate(zip(targets, output)): seen += 1 if detections is None: # If there are labels but no detections mark as zero AP if labels.size(0) != 0: mAPs.append(0), mR.append(0), mP.append(0) continue # Get detections sorted by decreasing confidence scores detections = detections.cpu().numpy() detections = detections[np.argsort(-detections[:, 4])] # If no labels add number of detections as incorrect correct = [] if labels.size(0) == 0: # correct.extend([0 for _ in range(len(detections))]) mAPs.append(0), mR.append(0), mP.append(0) continue else: target_cls = torch.zeros_like(labels[:, 0]) target_boxes = labels[:, 2:6] detected = [] for *pred_bbox, conf, obj_conf in detections: obj_pred = 0 pred_bbox = torch.FloatTensor(pred_bbox).view(1, -1) # Compute iou with target boxes iou = bbox_iou(pred_bbox, target_boxes, x1y1x2y2=True)[0] # Extract index of largest overlap best_i = np.argmax(iou) # If overlap exceeds threshold and classification is correct mark as correct if iou[best_i] > iou_thres and best_i not in detected: correct.append(1) detected.append(best_i) else: correct.append(0) # Compute Average Precision (AP) per class AP, AP_class, R, P = ap_per_class(tp=correct, conf=detections[:, 4], pred_cls=np.zeros_like(detections[:, 5]), # detections[:, 6] target_cls=target_cls) # Accumulate AP per class AP_accum_count += np.bincount(AP_class, minlength=nC) AP_accum += np.bincount(AP_class, minlength=nC, weights=AP) # Compute mean AP across all classes in this image, and append to image list mAPs.append(AP.mean()) mR.append(R.mean()) mP.append(P.mean()) # Means of all images mean_mAP = np.sum(mAPs) / ( AP_accum_count + 1E-16) mean_R = np.sum(mR) / ( AP_accum_count + 1E-16) mean_P = np.sum(mP) / (AP_accum_count + 1E-16) if batch_i % print_interval==0: # Print image mAP and running mean mAP print(('%11s%11s' + '%11.3g' * 4 + 's') % (seen, dataloader.dataset.nF, mean_P, mean_R, mean_mAP, time.time() - t)) # Print mAP per class print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) print('AP: %-.4f\n\n' % (AP_accum[0] / (AP_accum_count[0] + 1E-16))) # Return mAP return mean_mAP, mean_R, mean_P
def test(model, fetcher, conf_thres=1e-3, nms_thres=0.5): model.eval() val_loss = 0 classes = fetcher.loader.dataset.classes num_classes = len(classes) seen = 0 s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1') p, r, f1, mp, mr, mAP, mf1 = 0., 0., 0., 0., 0., 0., 0. jdict, stats, ap, ap_class = [], [], [], [] pbar = tqdm(enumerate(fetcher), total=len(fetcher)) for idx, (imgs, targets) in pbar: _, _, height, width = imgs.shape # batch size, channels, height, width # Run model inf_out, train_out = model(imgs) # inference and training outputs # Compute loss val_loss += compute_loss(train_out, targets, model).item() # GIoU, obj, cls # Run NMS output = non_max_suppression(inf_out, conf_thres=conf_thres, nms_thres=nms_thres) # Plot images with bounding boxes if idx == 0: show_batch(imgs, output) # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append(([], torch.Tensor(), torch.Tensor(), tcls)) continue # Clip boxes to image bounds clip_coords(pred, (height, width)) # Assign all predictions as incorrect correct = [0] * len(pred) if nl: detected = [] tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) tbox[:, [0, 2]] *= width tbox[:, [1, 3]] *= height # Search for correct predictions for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred): # Break if all targets already located in image if len(detected) == nl: break # Continue if predicted class not among image classes if pcls.item() not in tcls: continue # Best iou, index between pred and targets m = (pcls == tcls_tensor).nonzero().view(-1) iou, bi = bbox_iou(pbox, tbox[m]).max(0) # If iou > threshold and class is correct mark as correct if iou > 0.5 and m[ bi] not in detected: # and pcls == tcls[bi]: correct[i] = 1 detected.append(m[bi]) # Append statistics (correct, conf, pcls, tcls) stats.append( (correct, pred[:, 4].cpu().numpy(), pred[:, 6].cpu().numpy(), tcls)) pbar.set_description('loss: %8g' % (val_loss / (idx + 1))) # Compute statistics stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # sync stats if dist.is_initialized(): for i in range(len(stats)): stat = torch.FloatTensor(stats[i]).to(device) ls = torch.IntTensor([len(stat)]).to(device) ls_list = [ torch.IntTensor([0]).to(device) for _ in range(dist.get_world_size()) ] dist.all_gather(ls_list, ls) ls_list = [ls_item.item() for ls_item in ls_list] max_ls = max(ls_list) if len(stat) < max_ls: stat = torch.cat( [stat, torch.zeros(max_ls - len(stat)).to(device)]) stat_list = [ torch.zeros(max_ls).to(device) for _ in range(dist.get_world_size()) ] dist.all_gather(stat_list, stat) stat_list = [ stat_list[si][:ls_list[si]] for si in range(dist.get_world_size()) if ls_list[si] > 0 ] stat = torch.cat(stat_list) stats[i] = stat.cpu().numpy() if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) mp, mr, mAP, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=num_classes) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, mAP, mf1)) # Print results per class for i, c in enumerate(ap_class): print(pf % (classes[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Return results mAPs = np.zeros(num_classes) + mAP for i, c in enumerate(ap_class): mAPs[c] = ap[i] # return (mp, mr, mAP, mf1, *(loss / len(dataloader)).tolist()), mAPs return mAP
def test(cfg, data, batch_size, img_size, conf_thres, iou_thres, nms_thres, src_txt_path, weights, log_file_path=None, model=None): # 0、初始化一些参数 data = parse_data_cfg(data) nc = int(data['classes']) # number of classes names = load_classes(data['names']) # 1、加载网络 if model is None: device = select_device('0') model = Darknet(cfg) if weights.endswith('.pt'): # TODO: .weights权重格式 model.load_state_dict( torch.load(weights, map_location=device)['model'] ) # 20200704_50epoch_modify_noobj # TODO:map_location=device ? if torch.cuda.device_count() > 1: model = nn.DataParallel(model) # clw note: 多卡 else: device = next(model.parameters()).device # get model device model.to(device).eval() # 2、加载数据集 test_dataset = VocDataset(src_txt_path, img_size, with_label=True, is_training=False) dataloader = DataLoader( test_dataset, batch_size=batch_size, shuffle=False, num_workers=8, # TODO collate_fn=test_dataset.test_collate_fn, # TODO pin_memory=True) # 3、预测,前向传播 image_nums = 0 s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@{}'.format(iou_thres), 'F1') #s = ('%20s' + '%10s' * 6) % ('Class', 'ImgNum', 'Target', 'P', 'R', '[email protected]', 'F1') p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0. jdict, stats, ap, ap_class = [], [], [], [] pbar = tqdm(dataloader) for i, (img_tensor, target_tensor, _, _) in enumerate(pbar): img_tensor = img_tensor.to(device) # (bs, 3, 416, 416) target_tensor = target_tensor.to(device) height, width = img_tensor.shape[2:] start = time.time() # Disable gradients with torch.no_grad(): # (1) Run model output = model( img_tensor ) # (x1, y1, x2, y2, obj_conf, class_conf, class_pred) # (2) NMS nms_output = non_max_suppression(output, conf_thres, nms_thres) s = 'time use per batch: %.3fs' % (time.time() - start) pbar.set_description(s) for batch_idx, pred in enumerate(nms_output): # pred: (bs, 7) labels = target_tensor[target_tensor[:, 0] == batch_idx, 1:] nl = len(labels) # len of label tcls = labels[:, 0].tolist() if nl else [] # target class image_nums += 1 # 考虑一个预测 box 都没有的情况,比如 conf 太高 if pred is None: if nl: stats.append(([], torch.Tensor(), torch.Tensor(), tcls)) continue # Clip boxes to image bounds TODO:有必要,因为 label 都是经过clip的,所以如果去掉clip,mAP应该会有所降低 clip_coords(pred, (height, width)) # mAP is the same # Assign all predictions as incorrect correct = [0] * len(pred) if nl: detected = [] tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) tbox[:, [0, 2]] *= img_tensor[batch_idx].size()[2] # w tbox[:, [1, 3]] *= img_tensor[batch_idx].size()[1] # h # Search for correct predictions for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred): # Break if all targets already located in image if len(detected) == nl: break # Continue if predicted class not among image classes if pcls.item() not in tcls: continue # Best iou, index between pred and targets m = (pcls == tcls_tensor).nonzero().view(-1) iou, bi = bbox_iou(pbox, tbox[m]).max(0) # If iou > threshold and class is correct mark as correct if iou > iou_thres and m[ bi] not in detected: # and pcls == tcls[bi]: correct[i] = 1 detected.append(m[bi]) # print('stats.append: ', (correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls)) ''' pred flag ( [1, 0, 1, 0, 0, 1, 0, 0, 1], pred conf tensor([0.17245, 0.14642, 0.07215, 0.07138, 0.07069, 0.06449, 0.06222, 0.05580, 0.05452]), pred cls tensor([2., 2., 2., 2., 2., 2., 2., 2., 2.]), lb_cls [2.0, 2.0, 2.0, 2.0, 2.0]) stats is a [] ''' stats.append( (correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls)) # Append statistics (correct, conf, pcls, tcls) # after get stats for all images , ... # Compute statistics stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results # time.sleep(0.01) # clw note: 防止前面 tqdm 还没输出,但是这里已经打印了 #pf = '%20s' + '%10.3g' * 6 # print format pf = '%20s' + '%10s' + '%10.3g' * 5 pf_value = pf % ('all', str(image_nums), nt.sum(), mp, mr, map, mf1) print(pf_value) if __name__ != '__main__': write_to_file(s, log_file_path) write_to_file(pf_value, log_file_path) results = [] results.append({"all": (mp, mr, map, mf1)}) # Print results per class #if verbose and nc > 1 and len(stats): if nc > 1 and len(stats): for i, c in enumerate(ap_class): #print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) print(pf % (names[c], '', nt[c], p[i], r[i], ap[i], f1[i])) if __name__ != '__main__': write_to_file( pf % (names[c], '', nt[c], p[i], r[i], ap[i], f1[i]), log_file_path) results.append({names[c]: (p[i], r[i], ap[i], f1[i])}) # Return results maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1), maps
def main(): """Create a TensorRT engine for ONNX-based YOLOv3-608 and run inference.""" # Try to load a previously generated YOLOv3-608 network graph in ONNX format: onnx_file_path = './yolov3.onnx' engine_file_path = "yolov3.trt" data_path = "./data/unrel.data" data = parse_data_cfg(data_path) nc = int(data['classes']) # number of classes path = data['valid'] # path to test images names = load_classes(data['names']) # class names iouv = torch.linspace(0.5, 0.95, 1, dtype=torch.float32) # iou vector for [email protected]:0.95 niou = 1 conf_thres = 0.001 iou_thres = 0.6 verbose = True # Genearte custom dataloader img_size = 448 # copy form pytorch src batch_size = 16 dataset = LoadImagesAndLabels(path, img_size, batch_size, rect=True) batch_size = min(batch_size, len(dataset)) dataloader = data_loader(dataset, batch_size, img_size) # Output shapes expected by the post-processor output_shapes = [(16, 126, 14, 14), (16, 126, 28, 28), (16, 126, 56, 56)] # Do inference with TensorRT trt_outputs = [] with get_engine(onnx_file_path, engine_file_path ) as engine, engine.create_execution_context() as context: inputs, outputs, bindings, stream = common.allocate_buffers(engine) s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1') p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. pbar = tqdm.tqdm(dataloader, desc=s) stats, ap, ap_class = [], [], [] seen = 0 for batch_i, (imgs, targets, paths, shapes) in enumerate(pbar): imgs = imgs.astype(np.float32) / 255.0 nb, _, height, width = imgs.shape # batch size, channels, height, width whwh = np.array([width, height, width, height]) inputs[0].host = imgs postprocessor_args = { "yolo_masks": [ (6, 7, 8), (3, 4, 5), (0, 1, 2) ], # A list of 3 three-dimensional tuples for the YOLO masks "yolo_anchors": [ (10, 13), (16, 30), (33, 23), (30, 61), ( 62, 45 ), # A list of 9 two-dimensional tuples for the YOLO anchors (59, 119), (116, 90), (156, 198), (373, 326) ], "num_classes": 37, "stride": [32, 16, 8] } postprocessor = PostprocessYOLO(**postprocessor_args) # Do layers before yolo t = time.time() trt_outputs = common.do_inference_v2(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream) trt_outputs = [ output.reshape(shape) for output, shape in zip(trt_outputs, output_shapes) ] trt_outputs = [ np.ascontiguousarray( otpt[:, :, :int(imgs.shape[2] * (2**i) / 32), :int(imgs.shape[3] * (2**i) / 32)], dtype=np.float32) for i, otpt in enumerate(trt_outputs) ] output_list = postprocessor.process(trt_outputs) t0 += time.time() - t inf_out = torch.cat(output_list, 1) t = time.time() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # nms t1 += time.time() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh tbox = tbox.type(torch.float32) # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero().view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero().view( -1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[pi[j]] = ious[ j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # Plot images if batch_i < 1: f = 'test_batch%g_gt.jpg' % batch_i # filename plot_images(imgs, targets, paths=paths, names=names, fname=f) # ground truth f = 'test_batch%g_pred.jpg' % batch_i plot_images(imgs, output_to_target(output, width, height), paths=paths, names=names, fname=f) # predictions # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) if niou > 1: p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean( 1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]] mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # Print results pf = '%20s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Print speeds if verbose: t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + ( img_size, img_size, batch_size) # tuple print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
def _evaluate(model, dataloader, class_names, img_size, iou_thres, conf_thres, nms_thres, verbose): """Evaluate model on validation dataset. :param model: Model to evaluate :type model: models.Darknet :param dataloader: Dataloader provides the batches of images with targets :type dataloader: DataLoader :param class_names: List of class names :type class_names: [str] :param img_size: Size of each image dimension for yolo :type img_size: int :param iou_thres: IOU threshold required to qualify as detected :type iou_thres: float :param conf_thres: Object confidence threshold :type conf_thres: float :param nms_thres: IOU threshold for non-maximum suppression :type nms_thres: float :param verbose: If True, prints stats of model :type verbose: bool :return: Returns precision, recall, AP, f1, ap_class """ model.eval() # Set model to evaluation mode Tensor = torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) for _, imgs, targets in tqdm.tqdm(dataloader, desc="Validating"): # Extract labels labels += targets[:, 1].tolist() # Rescale target targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size imgs = Variable(imgs.type(Tensor), requires_grad=False) with torch.no_grad(): outputs = to_cpu(model(imgs)) outputs = non_max_suppression(outputs, conf_thres=conf_thres, iou_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, targets, iou_threshold=iou_thres) if len(sample_metrics) == 0: # No detections over whole validation set. print("---- No detections over whole validation set ----") return None # Concatenate sample statistics true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] metrics_output = ap_per_class(true_positives, pred_scores, pred_labels, labels) print_eval_stats(metrics_output, class_names, verbose) return metrics_output
def evaluate(model, dataset_des, thres, batch_size, init_dim_cluster, diagnosis_code=0): model.eval() iou_thres, conf_thres, nms_thres = thres # Get dataloader print("Begin loading validation dataset ......") t_load_data = time.time() dataset = torchvision.datasets.VOCDetection(root='data/VOC/', year=dataset_des[0], image_set=dataset_des[1], transforms=None, download=True) dataset_dict = trans_voc(dataset) dataset = ListDataset(dataset_dict) loader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, shuffle=False, pin_memory=True, collate_fn=dataset.collate_fn, ) print("Complete loading validation dataset in {} s".format(time.time() - t_load_data)) labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) for i_batch, (img_ind, img, raw_targets, transform_params, tar_boxes) in enumerate(loader): print("\n++++++++++ i_batch (val) {} ++++++++++".format(i_batch)) batch_step_counter = 0 # Extract labels: raw_targets -- [t_conf, t_cls, x, y, w, h, one_hot_t_cls(20)] labels += tar_boxes[:, 1].tolist() if len(img) != batch_size: print("Current batch size is smaller than opt.batch_size!") continue img = img.to('cuda') raw_targets = raw_targets.to('cuda') tar_boxes = tar_boxes.to('cuda') input_img = img with torch.no_grad(): pred_conf_cls = model(input_img) pred_conf_cls = pred_conf_cls.permute(0, 2, 3, 1) pred_conf = torch.sigmoid(pred_conf_cls[:, :, :, 0]) pred_cls = torch.sigmoid(pred_conf_cls[:, :, :, 1:]) obj_mask = pred_conf > conf_thres obj_mask = obj_mask.byte().to('cuda') if diagnosis_code == 0: pass if diagnosis_code == 1: # localization ground-truth pred_bbox = raw_targets[:, :, :, 2:6] # pred_conf_cls = pred_conf_cls.permute(0, 2, 3, 1) # pred_conf = torch.sigmoid(pred_conf_cls[:, :, :, 0]) # pred_cls = torch.sigmoid(pred_conf_cls[:, :, :, 1:]) if diagnosis_code == 2: # classification ground-truth pass if diagnosis_code == 3: # full ground-truth pred_bbox = raw_targets[:, :, :, 2:6] pred_conf = raw_targets[:, :, :, 0] pred_cls = raw_targets[:, :, :, 6:] # if i_batch < 20: # im = Image.open("data/VOC/VOCdevkit/VOC2007/JPEGImages/{}".format(img_ind[0])).convert('RGB') # plot_detection_result(im, img_ind, pred_bbox, pred_conf, pred_cls, transform_params, iou=0.9) pred_outputs = torch.cat((pred_bbox, pred_conf.unsqueeze(3), pred_cls), dim=3) b, w, h, d = pred_outputs.shape pred_outputs = pred_outputs.view(b, w * h, d) outputs = non_max_suppression(pred_outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs, tar_boxes, iou_threshold=iou_thres) # Concatenate sample statistics true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] precision, recall, AP, f1, ap_class = ap_per_class(true_positives, pred_scores, pred_labels, labels) return precision, recall, AP, f1, ap_class
def evaluate( model, path: str, transform, iou_thres: float = 0.5, conf_thres: float = 0.5, nms_thres: float = 0.5, img_size: int = 416, batch_size: int = 8, ): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.eval() dataset = ListDataset(list_path=path, transform=transform, img_size=img_size, num_samples=None) dataloader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=dataset.collate_fn, ) labels = [] sample_metrics = [] # List of tuples (TP, confs, pred) for batch_i, (_, imgs, targets) in enumerate( tqdm.tqdm(dataloader, desc="Detecting objects")): labels += targets[:, 1].tolist() targets[:, 2:] = xywh2xyxy(targets[:, 2:]) targets[:, 2:] *= img_size imgs = imgs.requires_grad_(False).to(device) with torch.no_grad(): outputs = model(imgs) outputs = non_max_suppression(prediction=outputs, conf_thres=conf_thres, nms_thres=nms_thres) sample_metrics += get_batch_statistics(outputs=outputs, targets=targets, iou_threshold=iou_thres) # Concatenate sample statistics if sample_metrics: true_positives, pred_scores, pred_labels = [ np.concatenate(x, 0) for x in list(zip(*sample_metrics)) ] else: true_positives, pred_scores, pred_labels = [ np.zeros(1), np.zeros(1), np.zeros(1) ] precision, recall, AP, f1, ap_class = ap_per_class(true_positives, pred_scores, pred_labels, labels) return precision, recall, AP, f1, ap_class