def evaluate(netname, path_imgrec, num_classes, num_seg_classes, mean_pixels, data_shape, model_prefix, epoch, ctx=mx.cpu(), batch_size=1, path_imglist="", nms_thresh=0.45, force_nms=False, ovp_thresh=0.5, use_difficult=False, class_names=None, seg_class_names=None, voc07_metric=False): """ evalute network given validation record file Parameters: ---------- net : str or None Network name or use None to load from json without modifying path_imgrec : str path to the record validation file path_imglist : str path to the list file to replace labels in record file, optional num_classes : int number of classes, not including background mean_pixels : tuple (mean_r, mean_g, mean_b) data_shape : tuple or int (3, height, width) or height/width model_prefix : str model prefix of saved checkpoint epoch : int load model epoch ctx : mx.ctx mx.gpu() or mx.cpu() batch_size : int validation batch size nms_thresh : float non-maximum suppression threshold force_nms : boolean whether suppress different class objects ovp_thresh : float AP overlap threshold for true/false postives use_difficult : boolean whether to use difficult objects in evaluation if applicable class_names : comma separated str class names in string, must correspond to num_classes if set voc07_metric : boolean whether to use 11-point evluation as in VOC07 competition """ global outimgiter # set up logger logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # args if isinstance(data_shape, int): data_shape = (3, data_shape, data_shape) else: data_shape = map(int, data_shape.split(",")) assert len(data_shape) == 3 and data_shape[0] == 3 model_prefix += '_' + str(data_shape[1]) # iterator eval_iter = MultiTaskRecordIter(path_imgrec, batch_size, data_shape, path_imglist=path_imglist, enable_aug=False, **cfg.valid) # model params load_net, args, auxs = mx.model.load_checkpoint(model_prefix, epoch) # network if netname is None: net = load_net elif netname.endswith("det"): net = get_det_symbol(netname.split("_")[0], data_shape[1], num_classes=num_classes, nms_thresh=nms_thresh, force_suppress=force_nms) elif netname.endswith("seg"): net = get_seg_symbol(netname.split("_")[0], data_shape[1], num_classes=num_classes, nms_thresh=nms_thresh, force_suppress=force_nms) elif netname.endswith("multi"): net = get_multi_symbol(netname.split("_")[0], data_shape[1], num_classes=num_classes, nms_thresh=nms_thresh, force_suppress=force_nms) else: raise NotImplementedError("") if not 'label_det' in net.list_arguments(): label_det = mx.sym.Variable(name='label_det') net = mx.sym.Group([net, label_det]) if not 'seg_out_label' in net.list_arguments(): seg_out_label = mx.sym.Variable(name='seg_out_label') net = mx.sym.Group([net, seg_out_label]) # init module # mod = mx.mod.Module(net, label_names=('label_det','seg_out_label',), logger=logger, context=ctx, # fixed_param_names=net.list_arguments()) # mod.bind(data_shapes=eval_iter.provide_data, label_shapes=eval_iter.provide_label) # mod.set_params(args, auxs, allow_missing=False, force_init=True) # metric = MApMetric(ovp_thresh, use_difficult, class_names) # results = mod.score(eval_iter, metric, num_batch=None) # for k, v in results: # print("{}: {}".format(k, v)) ctx = ctx[0] eval_metric = CustomAccuracyMetric() multibox_metric = MultiBoxMetric() depth_metric = DistanceAccuracyMetric(class_names=class_names) det_metric = MApMetric(ovp_thresh, use_difficult, class_names) seg_metric = IoUMetric(class_names=seg_class_names, axis=1) eval_metrics = metric.CompositeEvalMetric() eval_metrics.add(multibox_metric) eval_metrics.add(eval_metric) arg_params = {key: val.as_in_context(ctx) for key, val in args.items()} aux_params = {key: val.as_in_context(ctx) for key, val in auxs.items()} data_name = eval_iter.provide_data[0][0] label_name_det = eval_iter.provide_label[0][0] label_name_seg = eval_iter.provide_label[1][0] symbol = load_net # evaluation logger.info(" in eval process...") logger.info( str({ "ovp_thresh": ovp_thresh, "nms_thresh": nms_thresh, "batch_size": batch_size, "force_nms": force_nms, })) nbatch = 0 eval_iter.reset() eval_metrics.reset() det_metric.reset() total_time = 0 for data, fnames in eval_iter: nbatch += 1 label_shape_det = data.label[0].shape label_shape_seg = data.label[1].shape arg_params[data_name] = mx.nd.array(data.data[0], ctx) arg_params[label_name_det] = mx.nd.array(data.label[0], ctx) arg_params[label_name_seg] = mx.nd.array(data.label[1], ctx) executor = symbol.bind(ctx, arg_params, aux_states=aux_params) output_names = symbol.list_outputs() output_dict = dict(zip(output_names, executor.outputs)) cpu_output_array = mx.nd.zeros(output_dict["seg_out_output"].shape) ############## monitor status def stat_helper(name, array): """wrapper for executor callback""" import ctypes from mxnet.ndarray import NDArray from mxnet.base import NDArrayHandle, py_str array = ctypes.cast(array, NDArrayHandle) if 1: array = NDArray(array, writable=False).asnumpy() print(name, array.shape, np.mean(array), np.std(array), ('%.1fms' % (float(time.time() - stat_helper.start_time) * 1000))) else: array = NDArray(array, writable=False) array.wait_to_read() elapsed = float(time.time() - stat_helper.start_time) * 1000. if elapsed > 5: print(name, array.shape, ('%.1fms' % (elapsed, ))) stat_helper.start_time = time.time() stat_helper.start_time = float(time.time()) # executor.set_monitor_callback(stat_helper) ############## forward tic = time.time() executor.forward(is_train=True) output_dict["seg_out_output"].copyto(cpu_output_array) pred_shape = output_dict["seg_out_output"].shape label = mx.nd.array(data.label[1].reshape( (label_shape_seg[0], label_shape_seg[1] * label_shape_seg[2]))) output_dict["seg_out_output"].wait_to_read() toc = time.time() seg_out_output = output_dict["seg_out_output"].asnumpy() pred_seg_shape = output_dict["seg_out_output"].shape label_det = mx.nd.array(data.label[0].reshape( (label_shape_det[0], label_shape_det[1] * label_shape_det[2]))) label_seg = mx.nd.array(data.label[1].reshape( (label_shape_seg[0], label_shape_seg[1] * label_shape_seg[2])), ctx=ctx) pred_seg = mx.nd.array(output_dict["seg_out_output"].reshape( (pred_seg_shape[0], pred_seg_shape[1], pred_seg_shape[2] * pred_seg_shape[3])), ctx=ctx) #### remove invalid boxes out_det = output_dict["det_out_output"].asnumpy() indices = np.where(out_det[:, :, 0] >= 0) # labeled as negative out_det = np.expand_dims(out_det[indices[0], indices[1], :], axis=0) indices = np.where(out_det[:, :, 1] > .1) # higher confidence out_det = np.expand_dims(out_det[indices[0], indices[1], :], axis=0) # indices = np.where(out_det[:,:,6]<=(100/255.)) # too far away # out_det = np.expand_dims(out_det[indices[0],indices[1],:],axis=0) pred_det = mx.nd.array(out_det) #### remove labels too faraway # label_det = label_det.asnumpy().reshape((200,6)) # indices = np.where(label_det[:,5]<=(100./255.)) # label_det = np.expand_dims(label_det[indices[0],:],axis=0) # label_det = mx.nd.array(label_det) ################# display results #################### out_img = output_dict["seg_out_output"] out_img = mx.nd.split(out_img, axis=0, num_outputs=out_img.shape[0]) for imgidx in range(batch_size): seg_prob = out_img[imgidx] res_img = np.squeeze(seg_prob.asnumpy().argmax(axis=0).astype( np.uint8)) label_img = data.label[1].asnumpy()[imgidx, :, :].astype(np.uint8) img = np.squeeze(data.data[0].asnumpy()[imgidx, :, :, :]) det = out_det[imgidx, :, :] gt = label_det.asnumpy()[imgidx, :].reshape((-1, 6)) # save to results folder for evalutation res_fname = fnames[imgidx].replace("SegmentationClass", "results") lut = np.zeros(256) lut[:19] = np.array([ 7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33 ]) seg_resized = prob_upsampling(seg_prob, target_shape=(1024, 2048)) seg_resized2 = cv2.LUT(seg_resized, lut) # seg = cv2.LUT(res_img,lut) # cv2.imshow("seg",seg.astype(np.uint8)) cv2.imwrite(res_fname, seg_resized2) # display result print(fnames[imgidx], np.average(img)) display_img = display_results(res_img, np.expand_dims(label_img, axis=0), img, det, gt, class_names) res_fname = fnames[imgidx].replace("SegmentationClass", "output").replace( "labelTrainIds", "output") cv2.imwrite(res_fname, display_img) [exit(0) if (cv2.waitKey() & 0xff) == 27 else None] outimgiter += 1 ################# display results #################### eval_metrics.get_metric(0).update(None, [ output_dict["cls_prob_output"], output_dict["loc_loss_output"], output_dict["cls_label_output"] ]) eval_metrics.get_metric(1).update([label_seg], [pred_seg]) det_metric.update([mx.nd.slice_axis(data.label[0],axis=2,begin=0,end=5)], \ [mx.nd.slice_axis(pred_det,axis=2,begin=0,end=6)]) seg_metric.update([label_seg], [pred_seg]) disparities = [] for imgidx in range(batch_size): dispname = fnames[imgidx].replace("SegmentationClass", "Disparity").replace( "gtFine_labelTrainIds", "disparity") print(dispname) disparities.append(cv2.imread(dispname, -1)) depth_metric.update(mx.nd.array(disparities), [pred_det]) det_names, det_values = det_metric.get() seg_names, seg_values = seg_metric.get() depth_names, depth_values = depth_metric.get() total_time += toc - tic print("\r %d/%d %.1f%% speed=%.1fms %s=%.1f %s=%.1f %s=%.1f" % ( nbatch * eval_iter.batch_size, eval_iter.num_samples, float(nbatch * eval_iter.batch_size) * 100. / float(eval_iter.num_samples), total_time * 1000. / nbatch, det_names[-1], det_values[-1] * 100., seg_names[-1], seg_values[-1] * 100., depth_names[-1], depth_values[-1] * 100., ), end='\r') # if nbatch>50: break ## debugging names, values = eval_metrics.get() for name, value in zip(names, values): logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value) logger.info('----------------------------------------------') names, values = det_metric.get() for name, value in zip(names, values): logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value) logger.info('----------------------------------------------') logger.info(' & '.join(names)) logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values))) logger.info('----------------------------------------------') names, values = depth_metric.get() for name, value in zip(names, values): logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value) logger.info('----------------------------------------------') logger.info(' & '.join(names)) logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values))) logger.info('----------------------------------------------') names, values = seg_metric.get() for name, value in zip(names, values): logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value) logger.info('----------------------------------------------') logger.info(' & '.join(names)) logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values)))
def evaluate_net(net, imdb, mean_pixels, data_shape, model_prefix, epoch, ctx=mx.cpu(), batch_size=1, nms_thresh=0.45, force_nms=False, ovp_thresh=0.5, use_difficult=False, voc07_metric=False): """ evalute network given validation record file Parameters: ---------- net : str or None Network name or use None to load from json without modifying path_imgrec : str path to the record validation file path_imglist : str path to the list file to replace labels in record file, optional num_classes : int number of classes, not including background mean_pixels : tuple (mean_r, mean_g, mean_b) data_shape : tuple or int (3, height, width) or height/width model_prefix : str model prefix of saved checkpoint epoch : int load model epoch ctx : mx.ctx mx.gpu() or mx.cpu() batch_size : int validation batch size nms_thresh : float non-maximum suppression threshold force_nms : boolean whether suppress different class objects ovp_thresh : float AP overlap threshold for true/false postives use_difficult : boolean whether to use difficult objects in evaluation if applicable class_names : comma separated str class names in string, must correspond to num_classes if set voc07_metric : boolean whether to use 11-point evluation as in VOC07 competition """ # set up logger logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) num_classes = imdb.num_classes class_names = imdb.classes # args if isinstance(data_shape, int): data_shape = (3, data_shape, data_shape) assert len(data_shape) == 3 and data_shape[0] == 3 model_prefix += '_' + str(data_shape[1]) # iterator eval_iter = FaceTestIter(imdb, mean_pixels, img_stride=128, fix_hw=True) # model params load_net, args, auxs = mx.model.load_checkpoint(model_prefix, epoch) # network if net is None: net = load_net else: net = get_symbol(net, data_shape[1], num_classes=num_classes, nms_thresh=nms_thresh, force_suppress=force_nms) if not 'label' in net.list_arguments(): label = mx.sym.Variable(name='label') net = mx.sym.Group([net, label]) # init module mod = mx.mod.Module(net, label_names=('label',), logger=logger, context=ctx, fixed_param_names=net.list_arguments()) mod.bind(data_shapes=eval_iter.provide_data, label_shapes=eval_iter.provide_label) mod.set_params(args, auxs, allow_missing=False, force_init=True) # run evaluation if voc07_metric: metric = VOC07MApMetric(ovp_thresh, use_difficult, class_names) else: metric = MApMetric(ovp_thresh, use_difficult, class_names) results = [] for i, (datum, im_info) in enumerate(eval_iter): mod.reshape(data_shapes=datum.provide_data, label_shapes=datum.provide_label) mod.forward(datum) preds = mod.get_outputs() det0 = preds[0][0].asnumpy() # (n_anchor, 6) det0 = do_nms(det0, 1, nms_thresh) preds[0][0] = mx.nd.array(det0, ctx=preds[0].context) sy, sx, _ = im_info['im_shape'] scaler = mx.nd.array((1.0, sx, sy, sx, sy, 1.0)) scaler = mx.nd.reshape(scaler, (1, 1, -1)) datum.label[0] *= scaler metric.update(datum.label, preds) if i % 10 == 0: print('processed {} images.'.format(i)) # if i == 10: # break results = metric.get_name_value() for k, v in results: print("{}: {}".format(k, v))