def on_epoch_end(self, epoch, logs=None): """Compute the MAP of the validation set predictions.""" y_true, y_pred = self.predict() map_k = evaluation.compute_map(y_true, y_pred, self.k) map_k_min = min(evaluation.compute_map(y_true, y_pred, self.k, True)) # Log the computed value logs = logs or {} logs['val_map'] = map_k logs['val_map_min'] = map_k_min
def get_mAP(self): a = 0 if self.cat_flag == False: t = time.time() print('Runing per image evaluation........') for i in range(len(self.pred_boxes)): try: per_image = my_eval.compute_map( self.gt_boxes[i + 1], self.gt_class_ids[i + 1], self.pred_boxes[i + 1], self.pred_class_ids[i + 1], self.pred_scores[i + 1], iou_threshold=self.iou_threshold) except: continue b = per_image[0] F = per_image[1] self.eval['FP_bg'] += F['FP_bg'] self.eval['FP_cls'] += F['FP_cls'] self.eval['FN'] += F['FN'] # if b < 0.8: # print("Image:%d hasn't been fed up =.=" % i) a += b print('Accumulating evaluation results........') a = a / len(self.pred_boxes) self.eval['mAP'] = a print('Done (t=%fs).' % (time.time() - t)) print( "Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=all ] = %f" % a) else: IoU = np.arange(0.5, 1, 0.05) AP = np.zeros([10]) t = time.time() print('Runing per class evaluation........') for i in range(10): try: AP[i] = my_eval.compute_map_cat(self.gt_boxes_cat, self.pred_boxes_cat, self.pred_scores_cat, self.cat_num, IoU[i]) except: print('AP[%d] computation fails' % i) print('Accumulating evaluation results........') print('Done (t=%fs).' % (time.time() - t)) mAP = np.mean(AP) self.eval['mAP'] = mAP self.eval['mAP50'] = AP[0] self.eval['mAP75'] = AP[5] print( 'Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=all ] = %f' % mAP) print( 'Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=all ] = %f' % AP[0]) print( 'Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=all ] = %f' % AP[5])
def on_epoch_end(self, epoch, logs=None): """Compute the MAP of the validation set predictions.""" x, y_true = self.validation_data[:2] y_pred = self.model.predict(x) map_k = evaluation.compute_map(y_true, y_pred, self.k) # Log the computed value logs = logs or {} logs['val_map'] = map_k
def err_analysis(self): # unfinished print('Runing error analysis......') t = time.time() if self.cat_flag == True: self.fetch_data() for i in range(len(self.pred_boxes)): per_image = my_eval.compute_map(self.gt_boxes[i + 1], self.gt_class_ids[i + 1], self.pred_boxes[i + 1], self.pred_class_ids[i + 1], self.pred_scores[i + 1], iou_threshold=self.iou_threshold) b = per_image[0] F = per_image[1] self.eval['FP_bg'] += F['FP_bg'] self.eval['FP_cls'] += F['FP_cls'] self.eval['FN'] += F['FN'] print('Analysis finished (t=%fs)' % (time.time() - t)) print('Attention : threshold was designed as : %f' % self.threshold) print('Number of FP_cls : %d' % self.eval['FP_cls']) print('Number of FP_bg : %d' % self.eval['FP_bg']) print('Number of FN : %d' % self.eval['FN'])
def validate(sketch_dataloader, shape_dataloader, model, criterion, epoch, opt): """ test for one epoch on the testing set """ sketch_losses = utils.AverageMeter() sketch_top1 = utils.AverageMeter() shape_losses = utils.AverageMeter() shape_top1 = utils.AverageMeter() net_whole, net_bp, net_vp, net_ap, net_cls = model # optim_sketch, optim_shape, optim_centers = optimizer crt_cls, crt_tlc, w1, w2 = criterion net_whole.eval() net_bp.eval() net_vp.eval() net_ap.eval() net_cls.eval() sketch_features = [] sketch_scores = [] sketch_labels = [] shape_features = [] shape_scores = [] shape_labels = [] batch_time = utils.AverageMeter() end = time.time() for i, (sketches, k_labels) in enumerate(sketch_dataloader): sketches_v = Variable(sketches.cuda()) k_labels_v = Variable(k_labels.long().cuda()) sketch_feat = net_whole(sketches_v) sketch_score = net_cls(sketch_feat) loss = crt_cls(sketch_score, k_labels_v) prec1 = utils.accuracy(sketch_score.data, k_labels_v.data, topk=(1,))[0] sketch_losses.update(loss.data[0], sketch_score.size(0)) # batchsize sketch_top1.update(prec1[0], sketch_score.size(0)) sketch_features.append(sketch_feat.data.cpu()) sketch_labels.append(k_labels) sketch_scores.append(sketch_score.data.cpu()) batch_time.update(time.time() - end) end = time.time() if i % opt.print_freq == 0: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format( i, len(sketch_dataloader), batch_time=batch_time, loss=sketch_losses, top1=sketch_top1)) print(' *Sketch Prec@1 {top1.avg:.3f}'.format(top1=sketch_top1)) batch_time = utils.AverageMeter() end = time.time() for i, (shapes, p_labels) in enumerate(shape_dataloader): shapes = shapes.view(shapes.size(0)*shapes.size(1), shapes.size(2), shapes.size(3), shapes.size(4)) # expanding: (bz * 12) x 3 x 224 x 224 shapes = shapes.expand(shapes.size(0), 3, shapes.size(2), shapes.size(3)) shapes_v = Variable(shapes.cuda()) p_labels_v = Variable(p_labels.long().cuda()) o_bp = net_bp(shapes_v) o_vp = net_vp(o_bp) shape_feat = net_ap(o_vp) shape_score = net_cls(shape_feat) loss = crt_cls(shape_score, p_labels_v) prec1 = utils.accuracy(shape_score.data, p_labels_v.data, topk=(1,))[0] shape_losses.update(loss.data[0], shape_score.size(0)) # batchsize shape_top1.update(prec1[0], shape_score.size(0)) shape_features.append(shape_feat.data.cpu()) shape_labels.append(p_labels) shape_scores.append(shape_score.data.cpu()) batch_time.update(time.time() - end) end = time.time() if i % opt.print_freq == 0: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format( i, len(shape_dataloader), batch_time=batch_time, loss=shape_losses, top1=shape_top1)) print(' *Shape Prec@1 {top1.avg:.3f}'.format(top1=shape_top1)) shape_features = torch.cat(shape_features, 0).numpy() sketch_features = torch.cat(sketch_features, 0).numpy() shape_scores = torch.cat(shape_scores, 0).numpy() sketch_scores = torch.cat(sketch_scores, 0).numpy() shape_labels = torch.cat(shape_labels, 0).numpy() sketch_labels = torch.cat(sketch_labels, 0).numpy() # d = compute_distance(sketch_features.copy(), shape_features.copy(), l2=False) # scio.savemat('test/example.mat',{'d':d, 'feat':dataset_features, 'labels':dataset_labels}) # AUC, mAP = map_and_auc(sketch_labels.copy(), shape_labels.copy(), d) # print(' * Feature AUC {0:.5} mAP {0:.5}'.format(AUC, mAP)) d_feat = compute_distance(sketch_features.copy(), shape_features.copy(), l2=False) d_feat_norm = compute_distance(sketch_features.copy(), shape_features.copy(), l2=True) mAP_feat = compute_map(sketch_labels.copy(), shape_labels.copy(), d_feat) mAP_feat_norm = compute_map(sketch_labels.copy(), shape_labels.copy(), d_feat_norm) print(' * Feature mAP {0:.5%}\tNorm Feature mAP {1:.5%}'.format(mAP_feat, mAP_feat_norm)) d_score = compute_distance(sketch_scores.copy(), shape_scores.copy(), l2=False) mAP_score = compute_map(sketch_labels.copy(), shape_labels.copy(), d_score) d_score_norm = compute_distance(sketch_scores.copy(), shape_scores.copy(), l2=True) mAP_score_norm = compute_map(sketch_labels.copy(), shape_labels.copy(), d_score_norm) if opt.sf: shape_paths = [img[0] for img in shape_dataloader.dataset.shape_target_path_list] sketch_paths = [img[0] for img in sketch_dataloader.dataset.sketch_target_path_list] scio.savemat('{}/test_feat_temp.mat'.format(opt.checkpoint_folder), {'score_dist':d_score, 'score_dist_norm': d_score_norm, 'feat_dist': d_feat, 'feat_dist_norm': d_feat_norm,'sketch_features':sketch_features, 'sketch_labels':sketch_labels, 'sketch_scores': sketch_scores, 'shape_features':shape_features, 'shape_labels':shape_labels, 'sketch_paths':sketch_paths, 'shape_paths':shape_paths}) print(' * Score mAP {0:.5%}\tNorm Score mAP {1:.5%}'.format(mAP_score, mAP_score_norm)) return [sketch_top1.avg, shape_top1.avg, mAP_feat, mAP_feat_norm, mAP_score, mAP_score_norm]