def evaluate(netname,
path_imgrec,
num_classes,
num_seg_classes,
mean_pixels,
data_shape,
model_prefix,
epoch,
ctx=mx.cpu(),
batch_size=1,
path_imglist="",
nms_thresh=0.45,
force_nms=False,
ovp_thresh=0.5,
use_difficult=False,
class_names=None,
seg_class_names=None,
voc07_metric=False):
"""
evalute network given validation record file
Parameters:
----------
net : str or None
Network name or use None to load from json without modifying
path_imgrec : str
path to the record validation file
path_imglist : str
path to the list file to replace labels in record file, optional
num_classes : int
number of classes, not including background
mean_pixels : tuple
(mean_r, mean_g, mean_b)
data_shape : tuple or int
(3, height, width) or height/width
model_prefix : str
model prefix of saved checkpoint
epoch : int
load model epoch
ctx : mx.ctx
mx.gpu() or mx.cpu()
batch_size : int
validation batch size
nms_thresh : float
non-maximum suppression threshold
force_nms : boolean
whether suppress different class objects
ovp_thresh : float
AP overlap threshold for true/false postives
use_difficult : boolean
whether to use difficult objects in evaluation if applicable
class_names : comma separated str
class names in string, must correspond to num_classes if set
voc07_metric : boolean
whether to use 11-point evluation as in VOC07 competition
"""
global outimgiter
# set up logger
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# args
if isinstance(data_shape, int):
data_shape = (3, data_shape, data_shape)
else:
data_shape = map(int, data_shape.split(","))
assert len(data_shape) == 3 and data_shape[0] == 3
model_prefix += '_' + str(data_shape[1])
# iterator
eval_iter = MultiTaskRecordIter(path_imgrec,
batch_size,
data_shape,
path_imglist=path_imglist,
enable_aug=False,
**cfg.valid)
# model params
load_net, args, auxs = mx.model.load_checkpoint(model_prefix, epoch)
# network
if netname is None:
net = load_net
elif netname.endswith("det"):
net = get_det_symbol(netname.split("_")[0],
data_shape[1],
num_classes=num_classes,
nms_thresh=nms_thresh,
force_suppress=force_nms)
elif netname.endswith("seg"):
net = get_seg_symbol(netname.split("_")[0],
data_shape[1],
num_classes=num_classes,
nms_thresh=nms_thresh,
force_suppress=force_nms)
elif netname.endswith("multi"):
net = get_multi_symbol(netname.split("_")[0],
data_shape[1],
num_classes=num_classes,
nms_thresh=nms_thresh,
force_suppress=force_nms)
else:
raise NotImplementedError("")
if not 'label_det' in net.list_arguments():
label_det = mx.sym.Variable(name='label_det')
net = mx.sym.Group([net, label_det])
if not 'seg_out_label' in net.list_arguments():
seg_out_label = mx.sym.Variable(name='seg_out_label')
net = mx.sym.Group([net, seg_out_label])
# init module
# mod = mx.mod.Module(net, label_names=('label_det','seg_out_label',), logger=logger, context=ctx,
# fixed_param_names=net.list_arguments())
# mod.bind(data_shapes=eval_iter.provide_data, label_shapes=eval_iter.provide_label)
# mod.set_params(args, auxs, allow_missing=False, force_init=True)
# metric = MApMetric(ovp_thresh, use_difficult, class_names)
# results = mod.score(eval_iter, metric, num_batch=None)
# for k, v in results:
# print("{}: {}".format(k, v))
ctx = ctx[0]
eval_metric = CustomAccuracyMetric()
multibox_metric = MultiBoxMetric()
depth_metric = DistanceAccuracyMetric(class_names=class_names)
det_metric = MApMetric(ovp_thresh, use_difficult, class_names)
seg_metric = IoUMetric(class_names=seg_class_names, axis=1)
eval_metrics = metric.CompositeEvalMetric()
eval_metrics.add(multibox_metric)
eval_metrics.add(eval_metric)
arg_params = {key: val.as_in_context(ctx) for key, val in args.items()}
aux_params = {key: val.as_in_context(ctx) for key, val in auxs.items()}
data_name = eval_iter.provide_data[0][0]
label_name_det = eval_iter.provide_label[0][0]
label_name_seg = eval_iter.provide_label[1][0]
symbol = load_net
# evaluation
logger.info(" in eval process...")
logger.info(
str({
"ovp_thresh": ovp_thresh,
"nms_thresh": nms_thresh,
"batch_size": batch_size,
"force_nms": force_nms,
}))
nbatch = 0
eval_iter.reset()
eval_metrics.reset()
det_metric.reset()
total_time = 0
for data, fnames in eval_iter:
nbatch += 1
label_shape_det = data.label[0].shape
label_shape_seg = data.label[1].shape
arg_params[data_name] = mx.nd.array(data.data[0], ctx)
arg_params[label_name_det] = mx.nd.array(data.label[0], ctx)
arg_params[label_name_seg] = mx.nd.array(data.label[1], ctx)
executor = symbol.bind(ctx, arg_params, aux_states=aux_params)
output_names = symbol.list_outputs()
output_dict = dict(zip(output_names, executor.outputs))
cpu_output_array = mx.nd.zeros(output_dict["seg_out_output"].shape)
############## monitor status
def stat_helper(name, array):
"""wrapper for executor callback"""
import ctypes
from mxnet.ndarray import NDArray
from mxnet.base import NDArrayHandle, py_str
array = ctypes.cast(array, NDArrayHandle)
if 1:
array = NDArray(array, writable=False).asnumpy()
print(name, array.shape, np.mean(array), np.std(array),
('%.1fms' %
(float(time.time() - stat_helper.start_time) * 1000)))
else:
array = NDArray(array, writable=False)
array.wait_to_read()
elapsed = float(time.time() - stat_helper.start_time) * 1000.
if elapsed > 5:
print(name, array.shape, ('%.1fms' % (elapsed, )))
stat_helper.start_time = time.time()
stat_helper.start_time = float(time.time())
# executor.set_monitor_callback(stat_helper)
############## forward
tic = time.time()
executor.forward(is_train=True)
output_dict["seg_out_output"].copyto(cpu_output_array)
pred_shape = output_dict["seg_out_output"].shape
label = mx.nd.array(data.label[1].reshape(
(label_shape_seg[0], label_shape_seg[1] * label_shape_seg[2])))
output_dict["seg_out_output"].wait_to_read()
toc = time.time()
seg_out_output = output_dict["seg_out_output"].asnumpy()
pred_seg_shape = output_dict["seg_out_output"].shape
label_det = mx.nd.array(data.label[0].reshape(
(label_shape_det[0], label_shape_det[1] * label_shape_det[2])))
label_seg = mx.nd.array(data.label[1].reshape(
(label_shape_seg[0], label_shape_seg[1] * label_shape_seg[2])),
ctx=ctx)
pred_seg = mx.nd.array(output_dict["seg_out_output"].reshape(
(pred_seg_shape[0], pred_seg_shape[1],
pred_seg_shape[2] * pred_seg_shape[3])),
ctx=ctx)
#### remove invalid boxes
out_det = output_dict["det_out_output"].asnumpy()
indices = np.where(out_det[:, :, 0] >= 0) # labeled as negative
out_det = np.expand_dims(out_det[indices[0], indices[1], :], axis=0)
indices = np.where(out_det[:, :, 1] > .1) # higher confidence
out_det = np.expand_dims(out_det[indices[0], indices[1], :], axis=0)
# indices = np.where(out_det[:,:,6]<=(100/255.)) # too far away
# out_det = np.expand_dims(out_det[indices[0],indices[1],:],axis=0)
pred_det = mx.nd.array(out_det)
#### remove labels too faraway
# label_det = label_det.asnumpy().reshape((200,6))
# indices = np.where(label_det[:,5]<=(100./255.))
# label_det = np.expand_dims(label_det[indices[0],:],axis=0)
# label_det = mx.nd.array(label_det)
################# display results ####################
out_img = output_dict["seg_out_output"]
out_img = mx.nd.split(out_img, axis=0, num_outputs=out_img.shape[0])
for imgidx in range(batch_size):
seg_prob = out_img[imgidx]
res_img = np.squeeze(seg_prob.asnumpy().argmax(axis=0).astype(
np.uint8))
label_img = data.label[1].asnumpy()[imgidx, :, :].astype(np.uint8)
img = np.squeeze(data.data[0].asnumpy()[imgidx, :, :, :])
det = out_det[imgidx, :, :]
gt = label_det.asnumpy()[imgidx, :].reshape((-1, 6))
# save to results folder for evalutation
res_fname = fnames[imgidx].replace("SegmentationClass", "results")
lut = np.zeros(256)
lut[:19] = np.array([
7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
31, 32, 33
])
seg_resized = prob_upsampling(seg_prob, target_shape=(1024, 2048))
seg_resized2 = cv2.LUT(seg_resized, lut)
# seg = cv2.LUT(res_img,lut)
# cv2.imshow("seg",seg.astype(np.uint8))
cv2.imwrite(res_fname, seg_resized2)
# display result
print(fnames[imgidx], np.average(img))
display_img = display_results(res_img,
np.expand_dims(label_img, axis=0),
img, det, gt, class_names)
res_fname = fnames[imgidx].replace("SegmentationClass",
"output").replace(
"labelTrainIds", "output")
cv2.imwrite(res_fname, display_img)
[exit(0) if (cv2.waitKey() & 0xff) == 27 else None]
outimgiter += 1
################# display results ####################
eval_metrics.get_metric(0).update(None, [
output_dict["cls_prob_output"], output_dict["loc_loss_output"],
output_dict["cls_label_output"]
])
eval_metrics.get_metric(1).update([label_seg], [pred_seg])
det_metric.update([mx.nd.slice_axis(data.label[0],axis=2,begin=0,end=5)], \
[mx.nd.slice_axis(pred_det,axis=2,begin=0,end=6)])
seg_metric.update([label_seg], [pred_seg])
disparities = []
for imgidx in range(batch_size):
dispname = fnames[imgidx].replace("SegmentationClass",
"Disparity").replace(
"gtFine_labelTrainIds",
"disparity")
print(dispname)
disparities.append(cv2.imread(dispname, -1))
depth_metric.update(mx.nd.array(disparities), [pred_det])
det_names, det_values = det_metric.get()
seg_names, seg_values = seg_metric.get()
depth_names, depth_values = depth_metric.get()
total_time += toc - tic
print("\r %d/%d %.1f%% speed=%.1fms %s=%.1f %s=%.1f %s=%.1f" % (
nbatch * eval_iter.batch_size,
eval_iter.num_samples,
float(nbatch * eval_iter.batch_size) * 100. /
float(eval_iter.num_samples),
total_time * 1000. / nbatch,
det_names[-1],
det_values[-1] * 100.,
seg_names[-1],
seg_values[-1] * 100.,
depth_names[-1],
depth_values[-1] * 100.,
),
end='\r')
# if nbatch>50: break ## debugging
names, values = eval_metrics.get()
for name, value in zip(names, values):
logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value)
logger.info('----------------------------------------------')
names, values = det_metric.get()
for name, value in zip(names, values):
logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value)
logger.info('----------------------------------------------')
logger.info(' & '.join(names))
logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values)))
logger.info('----------------------------------------------')
names, values = depth_metric.get()
for name, value in zip(names, values):
logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value)
logger.info('----------------------------------------------')
logger.info(' & '.join(names))
logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values)))
logger.info('----------------------------------------------')
names, values = seg_metric.get()
for name, value in zip(names, values):
logger.info(' epoch[%d] Validation-%s=%f', epoch, name, value)
logger.info('----------------------------------------------')
logger.info(' & '.join(names))
logger.info(' & '.join(map(lambda v: '%.1f' % (v * 100., ), values)))
def evaluate_net(net, imdb, mean_pixels, data_shape,
model_prefix, epoch, ctx=mx.cpu(), batch_size=1,
nms_thresh=0.45, force_nms=False,
ovp_thresh=0.5, use_difficult=False,
voc07_metric=False):
"""
evalute network given validation record file
Parameters:
----------
net : str or None
Network name or use None to load from json without modifying
path_imgrec : str
path to the record validation file
path_imglist : str
path to the list file to replace labels in record file, optional
num_classes : int
number of classes, not including background
mean_pixels : tuple
(mean_r, mean_g, mean_b)
data_shape : tuple or int
(3, height, width) or height/width
model_prefix : str
model prefix of saved checkpoint
epoch : int
load model epoch
ctx : mx.ctx
mx.gpu() or mx.cpu()
batch_size : int
validation batch size
nms_thresh : float
non-maximum suppression threshold
force_nms : boolean
whether suppress different class objects
ovp_thresh : float
AP overlap threshold for true/false postives
use_difficult : boolean
whether to use difficult objects in evaluation if applicable
class_names : comma separated str
class names in string, must correspond to num_classes if set
voc07_metric : boolean
whether to use 11-point evluation as in VOC07 competition
"""
# set up logger
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
num_classes = imdb.num_classes
class_names = imdb.classes
# args
if isinstance(data_shape, int):
data_shape = (3, data_shape, data_shape)
assert len(data_shape) == 3 and data_shape[0] == 3
model_prefix += '_' + str(data_shape[1])
# iterator
eval_iter = FaceTestIter(imdb, mean_pixels, img_stride=128, fix_hw=True)
# model params
load_net, args, auxs = mx.model.load_checkpoint(model_prefix, epoch)
# network
if net is None:
net = load_net
else:
net = get_symbol(net, data_shape[1], num_classes=num_classes,
nms_thresh=nms_thresh, force_suppress=force_nms)
if not 'label' in net.list_arguments():
label = mx.sym.Variable(name='label')
net = mx.sym.Group([net, label])
# init module
mod = mx.mod.Module(net, label_names=('label',), logger=logger, context=ctx,
fixed_param_names=net.list_arguments())
mod.bind(data_shapes=eval_iter.provide_data, label_shapes=eval_iter.provide_label)
mod.set_params(args, auxs, allow_missing=False, force_init=True)
# run evaluation
if voc07_metric:
metric = VOC07MApMetric(ovp_thresh, use_difficult, class_names)
else:
metric = MApMetric(ovp_thresh, use_difficult, class_names)
results = []
for i, (datum, im_info) in enumerate(eval_iter):
mod.reshape(data_shapes=datum.provide_data, label_shapes=datum.provide_label)
mod.forward(datum)
preds = mod.get_outputs()
det0 = preds[0][0].asnumpy() # (n_anchor, 6)
det0 = do_nms(det0, 1, nms_thresh)
preds[0][0] = mx.nd.array(det0, ctx=preds[0].context)
sy, sx, _ = im_info['im_shape']
scaler = mx.nd.array((1.0, sx, sy, sx, sy, 1.0))
scaler = mx.nd.reshape(scaler, (1, 1, -1))
datum.label[0] *= scaler
metric.update(datum.label, preds)
if i % 10 == 0:
print('processed {} images.'.format(i))
# if i == 10:
# break
results = metric.get_name_value()
for k, v in results:
print("{}: {}".format(k, v))