def evaluate(self, snapshot_name=''):
current_device = cuda.get_device_from_id(self.args.gpu)
with current_device:
gt_data = []
pred_data = []
for i, batch in enumerate(
tqdm(self.data_iterator,
total=len(self.data_loader) // self.args.batchsize)):
image, gt_bboxes, gt_labels = batch[0]
gt_data.append((gt_bboxes, gt_labels))
# if self.args.gpu is not None:
# image = cuda.to_gpu(image, current_device)
with cuda.Device(self.args.gpu):
with configuration.using_config('train', False):
bboxes, labels, scores = self.model.predict(
image.copy()[None, ...])
if len(bboxes[0]) == 0:
bboxes = [np.zeros((1, 4), dtype=np.float32)]
labels = [np.zeros((1, ), dtype=np.int32)]
scores = [np.zeros((1, ), dtype=np.float32)]
pred_data.append((bboxes[0], labels[0], scores[0]))
# TODO handle empty predictions!!
bboxes, labels, scores = zip(*pred_data)
gt_bboxes, gt_labels = concat_examples(gt_data)
result = eval_detection_voc(bboxes, labels, scores, gt_bboxes,
gt_labels, None)
map = result['map']
self.save_eval_results(snapshot_name, map)
def test_eval_detection_voc(self):
pred_bboxes = [
np.array([[0., 4., 1., 5.], [0., 0., 1., 1.]]),
np.array([[0., 0., 2., 2.], [2., 2., 3., 3.], [5., 5., 7., 7.]])
]
pred_labels = [np.array([0, 0]), np.array([0, 1, 1])]
pred_scores = [np.array([1., 0.9]), np.array([0.7, 0.6, 0.8])]
gt_bboxes = [
np.array([[0., 0., 1., 1.], [1., 0., 4., 4.]]),
np.array([[2., 2., 3., 3.]])
]
gt_labels = [np.array([0, 0]), np.array([1])]
results = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
iou_thresh=self.iou_thresh,
use_07_metric=self.use_07_metric)
np.testing.assert_equal(results[0]['recall'], self.rec0)
np.testing.assert_equal(results[0]['precision'], self.prec0)
np.testing.assert_almost_equal(results[0]['ap'], self.ap0)
np.testing.assert_equal(results[1]['recall'], self.rec1)
np.testing.assert_equal(results[1]['precision'], self.prec1)
np.testing.assert_almost_equal(results[1]['ap'], self.ap1)
np.testing.assert_almost_equal(results['map'],
(self.ap0 + self.ap1) / 2)
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
pred_bboxes, pred_labels, pred_scores, gt_values =\
apply_detection_link(target, it)
if len(gt_values) == 3:
gt_bboxes, gt_labels, gt_difficults = gt_values
elif len(gt_values) == 2:
gt_bboxes, gt_labels = gt_values
gt_difficults = None
eval_ = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=self.use_07_metric)
observation = {}
with reporter.report_scope(observation):
reporter.report({'map': eval_['map']}, target)
return observation
def main():
config, args = parse_args()
model = get_model(config["model"])
if args.gpu != -1:
model.to_gpu(args.gpu)
dataset = VOCBboxDataset(
data_dir="../dataset/VOC_test/VOC2007_test",
year='2007', split='test', use_difficult=True, return_difficult=True)
iterator = iterators.SerialIterator(
dataset, args.batchsize, repeat=False, shuffle=False)
imgs, pred_values, gt_values = apply_prediction_to_iterator(
model.predict, iterator, hook=ProgressHook(len(dataset)))
# delete unused iterator explicitly
del imgs
pred_bboxes, pred_labels, pred_scores = pred_values
gt_bboxes, gt_labels, gt_difficults = gt_values
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=True)
print()
print('mAP: {:f}'.format(result['map']))
for l, name in enumerate(voc_bbox_label_names):
if result['ap'][l]:
print('{:s}: {:f}'.format(name, result['ap'][l]))
else:
print('{:s}: -'.format(name))
def main():
chainer.config.train = False
parser = argparse.ArgumentParser()
parser.add_argument('--root', required=True)
parser.add_argument('--data_type', choices=opt.data_types, required=True)
parser.add_argument('--det_type',
choices=opt.detectors,
required=True,
default='ssd300')
parser.add_argument('--load', help='load original trained model')
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--batchsize', type=int, default=32)
args = parser.parse_args()
model_args = {
'n_fg_class': len(voc_bbox_label_names),
'pretrained_model': 'voc0712'
}
model = helper.get_detector(args.det_type, model_args)
if args.load:
chainer.serializers.load_npz(args.load, model)
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
model.use_preset('evaluate')
dataset = helper.get_detection_dataset(args.data_type, 'test', args.root)
iterator = chainer.iterators.SerialIterator(dataset,
args.batchsize,
repeat=False,
shuffle=False)
imgs, pred_values, gt_values = apply_prediction_to_iterator(
model.predict, iterator, hook=ProgressHook(len(dataset)))
# delete unused iterator explicitly
del imgs
pred_bboxes, pred_labels, pred_scores = pred_values
gt_bboxes, gt_labels = gt_values
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
use_07_metric=True)
aps = result['ap']
aps = aps[~np.isnan(aps)]
print('')
print('mAP: {:f}'.format(100.0 * result['map']))
print(aps)
def evaulate_single_model(self, logsdir):
metrics_file = join([logsdir, 'metrics.list'])
assert file_exists(
metrics_file), "Metrics file {} doesn't exist!".format(
metrics_file)
with open(join([logsdir, 'metrics.list']), 'rb') as f:
metrics = pickle.load(f)
p_b, p_l, p_s, r_b, g_b, g_l, m_b = zip(*metrics.values())
pred_bboxes, pred_labels, pred_scores, refn_bboxes, gt_bboxes, gt_labels, mtsm_bboxes = self.get_list_of_metrics(
[p_b, p_l, p_s, r_b, g_b, g_l, m_b])
for iou_thresh in np.linspace(0.5, 1.0, 11):
detector_stats = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
iou_thresh=iou_thresh)
bbox_align_stats = eval_detection_voc(refn_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
iou_thresh=iou_thresh)
straddling_stats = eval_detection_voc(mtsm_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
iou_thresh=iou_thresh)
sstr = self.pprint(detector_stats, bbox_align_stats,
straddling_stats, iou_thresh)
with open(
join([logsdir, 'metrics_{:.2f}.table'.format(iou_thresh)]),
'w') as f:
f.write(sstr)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model',
choices=('faster_rcnn', 'ssd300', 'ssd512'),
default='ssd300')
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--batchsize', type=int, default=32)
args = parser.parse_args()
if args.model == 'faster_rcnn':
model = FasterRCNNVGG16(pretrained_model='voc07')
elif args.model == 'ssd300':
model = SSD300(pretrained_model='voc0712')
elif args.model == 'ssd512':
model = SSD512(pretrained_model='voc0712')
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
model.use_preset('evaluate')
dataset = VOCDetectionDataset(year='2007',
split='test',
use_difficult=True,
return_difficult=True)
iterator = iterators.SerialIterator(dataset,
args.batchsize,
repeat=False,
shuffle=False)
pred_bboxes, pred_labels, pred_scores, gt_values = \
apply_detection_link(model, iterator, hook=ProgressHook(len(dataset)))
gt_bboxes, gt_labels, gt_difficults = gt_values
eval_ = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=True)
print()
print('mAP: {:f}'.format(eval_['map']))
for l, name in enumerate(voc_detection_label_names):
if l in eval_:
print('{:s}: {:f}'.format(name, eval_[l]['ap']))
else:
print('{:s}: -'.format(name))
def evaluate(self):
target = self._targets['main']
iterator = self._iterators['main']
iterator.reset()
ret = _apply_iterator(iterator, target)
result = eval_detection_voc(*ret)
report = {'map': result['map']}
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def eval_(out_values, rest_values):
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_difficults = rest_values
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=True)
print()
print('mAP: {:f}'.format(result['map']))
for l, name in enumerate(voc_bbox_label_names):
if result['ap'][l]:
print('{:s}: {:f}'.format(name, result['ap'][l]))
else:
print('{:s}: -'.format(name))
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
# set current device to devices['main']
with chainer.cuda.Device(self.device):
print(self.device)
imgs, pred_values, gt_values = apply_prediction_to_iterator(
target.predict, it)
# delete unused iterator explicitly
del imgs
pred_bboxes, pred_labels, pred_scores = pred_values
if len(gt_values) == 3:
gt_bboxes, gt_labels, gt_difficults = gt_values
elif len(gt_values) == 2:
gt_bboxes, gt_labels = gt_values
gt_difficults = None
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=self.use_07_metric)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = dict()
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def test_eval_detection_voc(self):
pred_bboxes = self.result['bboxes']
pred_labels = self.result['labels']
pred_scores = self.result['scores']
gt_bboxes = self.dataset['bboxes']
gt_labels = self.dataset['labels']
gt_difficults = self.dataset['difficults']
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=True)
# these scores were calculated by MATLAB code
expected = [
0.772727,
0.738780,
0.957576,
0.640153,
0.579473,
1.000000,
0.970030,
1.000000,
0.705931,
0.678719,
0.863636,
1.000000,
1.000000,
0.561364,
0.798813,
0.712121,
0.939394,
0.563636,
0.927273,
0.654545,
]
np.testing.assert_almost_equal(result['ap'], expected, decimal=5)
np.testing.assert_almost_equal(result['map'],
np.nanmean(expected),
decimal=5)
def evaluate(self):
target = self._targets['main']
if self.comm is not None and self.comm.rank != 0:
apply_to_iterator(target.predict, None, comm=self.comm)
return {}
iterator = self._iterators['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it, comm=self.comm)
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
if len(rest_values) == 3:
gt_bboxes, gt_labels, gt_difficults = rest_values
elif len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_difficults = None
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=self.use_07_metric)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def test_eval_detection_voc_one_bbox(self):
pred_bboxes = [
np.array([[0., 0., 1., 1.], [0., 0., 2., 2.], [0.3, 0.3, 0.5,
0.5]])
]
pred_labels = [np.array([0, 0, 0])]
pred_scores = [np.array([0.8, 0.9, 1.])]
gt_bboxes = [np.array([[0., 0., 1., 0.9]])]
gt_labels = [np.array([0])]
# iou is [0.95, 0.422, 0.3789]
results = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
iou_thresh=self.iou_thresh)
np.testing.assert_equal(results[0]['recall'], self.rec)
np.testing.assert_equal(results[0]['precision'], self.prec)
def test_eval_detection_voc(self):
pred_bboxes = self.result['bboxes']
pred_labels = self.result['labels']
pred_scores = self.result['scores']
gt_bboxes = self.dataset['bboxes']
gt_labels = self.dataset['labels']
gt_difficults = self.dataset['difficults']
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=True)
# these scores were calculated by MATLAB code
expected = [
0.772727,
0.738780,
0.957576,
0.640153,
0.579473,
1.000000,
0.970030,
1.000000,
0.705931,
0.678719,
0.863636,
1.000000,
1.000000,
0.561364,
0.798813,
0.712121,
0.939394,
0.563636,
0.927273,
0.654545,
]
np.testing.assert_almost_equal(result['ap'], expected, decimal=5)
np.testing.assert_almost_equal(
result['map'], np.nanmean(expected), decimal=5)
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
if len(rest_values) == 3:
gt_bboxes, gt_labels, gt_difficults = rest_values
elif len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_difficults = None
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=self.use_07_metric)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def test_eval_detection_voc_difficult(self):
pred_bboxes = [
np.array([[0., 0., 1., 1.], [0., 0., 2., 2.], [0.3, 0.3, 0.5,
0.5]])
]
pred_labels = [np.array([0, 0, 0])]
pred_scores = [np.array([0.8, 0.9, 1.])]
gt_bboxes = [np.array([[0., 0., 1., 0.9], [1., 1., 2., 2.]])]
gt_labels = [np.array([0, 0])]
gt_difficults = [np.array([False, True])]
# iou is [0.95, 0.422, 0.3789] and [0.142, 0.444, 0.048]
results = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults=gt_difficults,
iou_thresh=self.iou_thresh)
np.testing.assert_equal(results[0]['recall'], self.rec)
np.testing.assert_equal(results[0]['precision'], self.prec)
def calc(self):
pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels = self.data
if self.metric == 'all' or self.metric == 'voc_detection':
print('Calculating voc_detection ...')
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
use_07_metric=True)
print('mAP: {:f}'.format(result['map']))
print('person mAP: {:f}'.format(
result['ap'][voc_utils.voc_bbox_label_names.index('person')]))
if self.metric == 'all' or self.metric == 'pr_voc_detection':
print('Calculating pr_voc_detection ...')
prec, rec = calc_detection_voc_prec_rec(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults=None,
iou_thresh=0.5)
person_prec = prec[voc_utils.voc_bbox_label_names.index('person')]
person_rec = rec[voc_utils.voc_bbox_label_names.index('person')]
print('Avg person precision: {:f}'.format(np.average(person_prec)))
print('Avg person recall: {:f}'.format(np.average(person_rec)))
if self.plottings:
self.plot(recall=person_rec,
precision=person_prec,
metric='pr_voc_detection')
if self.metric == 'all' or self.metric == 'mot':
print('Calculating mot_metrics ...')
self.motmetrics()
def eval(Res,m=4952,F=True):
pred_bboxes=[]
pred_labels=[]
pred_scores=[]
gt_bboxes=[]
gt_labels=[]
names=GT.keys()
names=sorted(names)
for name in names[:m]:
res = Res[name]
res=res[res[:,4]>1e-10]
p_bboxes=res[:,:4]
p_labels=res[:,5]
p_labels=p_labels.astype(np.int32)
p_scores=res[:,4]
pred_bboxes.append(p_bboxes)
pred_labels.append(p_labels)
pred_scores.append(p_scores)
gt=GT[name]
gt=gt[gt[:,-1]==0]
g_bboxes = gt[:,:4]
g_labels=gt[:,4]-1
g_labels=g_labels.astype(np.int32)
gt_bboxes.append(g_bboxes)
gt_labels.append(g_labels)
result=eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels,use_07_metric=F)
print(result)
def evaluate(self,
predictions,
targets,
metric_list=['ap', 'map'],
eval_cfg=None):
result = eval_detection_voc(
pred_bboxes=predictions['pred_bboxes'],
pred_labels=predictions['pred_labels'],
pred_scores=predictions['pred_scores'],
gt_bboxes=targets['gt_bboxes'],
gt_labels=targets['gt_labels'],
iou_thresh=eval_cfg['iou_thresh'],
use_07_metric=eval_cfg['use_07_metric'],
)
result_selected = {}
for metric in metric_list:
result_selected[metric] = result[metric]
if 'ap' in result_selected:
ap_list = result_selected['ap']
ap_dict = {}
for idx, item in enumerate(ap_list):
ap_dict[self.classnames[idx]] = item
result_selected['ap'] = ap_dict
return result_selected
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(target, 'prediction_detection'):
target.prediction_detection = self.detection
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
del in_values
if self.detection:
# TODO: evaluate the detection result
pred_bboxes, pred_labels, pred_scores = out_values
if len(rest_values) == 3:
gt_bboxes, gt_labels, gt_difficults = rest_values
elif len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_difficults = None
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=self.use_07_metric)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
else:
# TODO: evaluate the segmentation result
pred_labels, = out_values
gt_labels, = rest_values
result = eval_semantic_segmentation(pred_labels, gt_labels)
report = {
'miou': result['miou'],
'pixel_accuracy': result['pixel_accuracy'],
'mean_class_accuracy': result['mean_class_accuracy']
}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['iou/{:s}'.format(
label_name)] = result['iou'][l]
report['class_accuracy/{:s}'.format(label_name)] = \
result['class_accuracy'][l]
except IndexError:
report['iou/{:s}'.format(label_name)] = np.nan
report['class_accuracy/{:s}'.format(
label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def main():
groundtruthpath = ARGS['groundtruthpath']
predictpath = ARGS['predictpath']
imagespath = ARGS['imagespath']
if (groundtruthpath == '' or predictpath == ''):
raise Exception('missing pti database paths')
#default yolo labeling format.
groundtruths = glob.glob(os.path.join(groundtruthpath, '**/*.txt'),
recursive=True)
#darkflow prediction output.
predictions = glob.glob(os.path.join(predictpath, '**/*.json'),
recursive=True)
gt_size = len(groundtruths)
pred_size = len(predictions)
print("{} prediction files | {} groundtruth files".format(
pred_size, gt_size))
pred_bboxes = []
pred_labels = []
pred_scores = []
gt_bboxes = []
gt_labels = []
images = []
for pred_file_path in predictions:
# print(pred_file_path)
#remove heading
base_pred_file_path = pred_file_path.replace(predictpath, '')
# print(base_pred_file_path)
gt_file_path = os.path.join(groundtruthpath,
base_pred_file_path).replace(
'.json', '.txt')
if not os.path.exists(gt_file_path):
print('GT file does not exist. Tried to open [{}]'.format(
gt_file_path))
sys.exit()
with open(gt_file_path) as gt_file:
bboxes = []
labels = []
for index, line in enumerate(gt_file):
if index == 0:
continue
gt = list(map(int,
line.split())) #[min_x, min_y, max_x, max_y]
#must be ('ymin', 'xmin', 'ymax', 'xmax')
bboxes.append([gt[1], gt[0], gt[3], gt[2]])
labels.append(voc_utils.voc_bbox_label_names.index('person'))
# print('bboxes',bboxes)
if len(bboxes) > 0:
bboxes = np.stack(bboxes).astype(np.float32)
labels = np.stack(labels).astype(np.int32)
else:
bboxes = np.ndarray(shape=(0), dtype=np.float32)
labels = np.ndarray(shape=(0), dtype=np.int32)
if imagespath:
images.append(
os.path.join(imagespath,
base_pred_file_path).replace('.json', '.jpg'))
gt_bboxes.append(bboxes)
gt_labels.append(labels)
with open(pred_file_path) as pred_file:
j_data = json.load(pred_file)
# print(j_data)
bboxes = []
labels = []
scores = []
for predicted_obj in j_data:
#must be ('ymin', 'xmin', 'ymax', 'xmax')
if predicted_obj['label'] != 'person':
continue
bboxes.append([
predicted_obj['topleft']['y'],
predicted_obj['topleft']['x'],
predicted_obj['bottomright']['y'],
predicted_obj['bottomright']['x']
])
labels.append(voc_utils.voc_bbox_label_names.index('person'))
scores.append(predicted_obj['confidence'])
if len(bboxes) > 0:
bboxes = np.stack(bboxes).astype(np.float32)
labels = np.stack(labels).astype(np.int32)
scores = np.stack(scores).astype(np.float32)
else:
bboxes = np.ndarray(shape=(0), dtype=np.float32)
labels = np.ndarray(shape=(0), dtype=np.int32)
scores = np.ndarray(shape=(0), dtype=np.float32)
pred_bboxes.append(bboxes)
pred_labels.append(labels)
pred_scores.append(scores)
# pred_bboxes = np.stack(pred_bboxes).astype(np.float32)
# pred_labels = np.stack(pred_labels).astype(np.int32)
# pred_scores = np.stack(pred_scores).astype(np.float32)
pred_bboxes = np.asarray(pred_bboxes)
pred_labels = np.asarray(pred_labels)
pred_scores = np.asarray(pred_scores)
gt_bboxes = np.asarray(gt_bboxes)
gt_labels = np.asarray(gt_labels)
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
use_07_metric=True)
if imagespath:
print('Gerenrating plots...')
for x in tqdm(range(100), total=100):
target_image = random.randint(0, len(gt_bboxes) - 1)
curr_img = images[target_image]
im = np.array(Image.open(curr_img), dtype=np.uint8)
fig, ax = plt.subplots(1)
ax.imshow(im)
for b in gt_bboxes[target_image]:
rect = buildRect(b, 'g')
ax.add_patch(rect)
for b in pred_bboxes[target_image]:
rect = buildRect(b, 'r')
ax.add_patch(rect)
plt.axis('off')
fig_dir = 'plottings'
os.makedirs(fig_dir, exist_ok=True)
fig_path = os.path.join(
fig_dir,
curr_img.replace(imagespath, '').replace('/', ''))
plt.savefig(fig_path, bbox_inches='tight')
plt.show()
plt.close()
print('mAP: {:f}'.format(result['map']))
print('person: {:f}'.format(
result['ap'][voc_utils.voc_bbox_label_names.index('person')]))
def evaluate(model, testdl, log, stop=100):
total_pred_boxes = []
total_pred_labels = []
total_pred_scores = []
total_true_boxes = []
total_true_labels = []
scales = len(cfg.DATA.SCALES)
n_i = 2 * scales
k = cfg.K
stop_flag = 0
with torch.no_grad():
model.eval()
for n_imgs, gt, n_regions, region in tqdm(testdl, "Evaluation"):
stop_flag += 1
if stop_flag > stop:
break
region = region.to(cfg.DEVICE)
avg_scores = torch.zeros((len(region[0]), 20), dtype=torch.float32)
for i in range(n_i):
per_img = n_imgs[i].to(cfg.DEVICE)
per_region = n_regions[i].to(cfg.DEVICE)
ref_scores1, ref_scores2, ref_scores3, proposal_scores = model(
per_img, per_region)
avg_scores += (ref_scores1 + ref_scores2 +
ref_scores3)[:, :-1].detach().cpu() / k
avg_scores /= n_i
gt = gt.numpy()[0]
gt_boxex = gt[:, :4]
gt_labels = gt[:, -1]
per_pred_boxes = []
per_pred_scores = []
per_pred_labels = []
region = region[0].cpu()
for i in range(20):
cls_scores = avg_scores[:, i]
cls_region = region
nms_filter = nms(cls_region, cls_scores, 0.3)
per_pred_boxes.append(cls_region[nms_filter].numpy())
per_pred_scores.append(cls_scores[nms_filter].numpy())
per_pred_labels.append(
np.full(len(nms_filter), i, dtype=np.int32))
total_pred_boxes.append(np.concatenate(per_pred_boxes, axis=0))
total_pred_scores.append(np.concatenate(per_pred_scores, axis=0))
total_pred_labels.append(np.concatenate(per_pred_labels, axis=0))
total_true_boxes.append(gt_boxex)
total_true_labels.append(gt_labels)
result = eval_detection_voc(
total_pred_boxes,
total_pred_labels,
total_pred_scores,
total_true_boxes,
total_true_labels,
iou_thresh=0.5,
use_07_metric=True,
)
print(f"Avg AP: {result['ap']}")
print(f"Avg mAP: {result['map']}")
write_log(log, f"Avg AP: {result['ap']}")
write_log(log, f"Avg mAP: {result['map']}")
return result['map']
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model',
choices=('faster_rcnn', 'ssd300', 'ssd512', 'yolo_v3'),
default='ssd300')
parser.add_argument('--pretrained_model')
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--batchsize', type=int, default=32)
args = parser.parse_args()
if args.model == 'faster_rcnn':
if args.pretrained_model:
model = FasterRCNNVGG16(n_fg_class=len(voc_bbox_label_names),
pretrained_model=args.pretrained_model)
else:
model = FasterRCNNVGG16(pretrained_model='voc07')
elif args.model == 'ssd300':
if args.pretrained_model:
model = SSD300(n_fg_class=len(voc_bbox_label_names),
pretrained_model=args.pretrained_model)
else:
model = SSD300(pretrained_model='voc0712')
elif args.model == 'ssd512':
if args.pretrained_model:
model = SSD512(n_fg_class=len(voc_bbox_label_names),
pretrained_model=args.pretrained_model)
else:
model = SSD512(pretrained_model='voc0712')
elif args.model == 'yolo_v3':
if args.pretrained_model:
model = YOLOv3(n_fg_class=len(voc_bbox_label_names),
pretrained_model=args.pretrained_model)
else:
model = YOLOv3(pretrained_model='voc0712')
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
model.use_preset('evaluate')
dataset = VOCBboxDataset(year='2007',
split='test',
use_difficult=True,
return_difficult=True)
iterator = iterators.SerialIterator(dataset,
args.batchsize,
repeat=False,
shuffle=False)
in_values, out_values, rest_values = apply_to_iterator(model.predict,
iterator,
hook=ProgressHook(
len(dataset)))
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_difficults = rest_values
result = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=True)
print()
print('mAP: {:f}'.format(result['map']))
for l, name in enumerate(voc_bbox_label_names):
if result['ap'][l]:
print('{:s}: {:f}'.format(name, result['ap'][l]))
else:
print('{:s}: -'.format(name))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model', choices=('faster_rcnn', 'ssd300', 'ssd512'),
default='ssd300')
parser.add_argument('--pretrained_model')
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--batchsize', type=int, default=32)
args = parser.parse_args()
if args.model == 'faster_rcnn':
if args.pretrained_model:
model = FasterRCNNVGG16(
n_fg_class=20,
pretrained_model=args.pretrained_model)
else:
model = FasterRCNNVGG16(pretrained_model='voc07')
elif args.model == 'ssd300':
if args.pretrained_model:
model = SSD300(
n_fg_class=20,
pretrained_model=args.pretrained_model)
else:
model = SSD300(pretrained_model='voc0712')
elif args.model == 'ssd512':
if args.pretrained_model:
model = SSD512(
n_fg_class=20,
pretrained_model=args.pretrained_model)
else:
model = SSD512(pretrained_model='voc0712')
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
model.use_preset('evaluate')
dataset = VOCBboxDataset(
year='2007', split='test', use_difficult=True, return_difficult=True)
iterator = iterators.SerialIterator(
dataset, args.batchsize, repeat=False, shuffle=False)
in_values, out_values, rest_values = apply_to_iterator(
model.predict, iterator, hook=ProgressHook(len(dataset)))
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_difficults = rest_values
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=True)
print()
print('mAP: {:f}'.format(result['map']))
for l, name in enumerate(voc_bbox_label_names):
if result['ap'][l]:
print('{:s}: {:f}'.format(name, result['ap'][l]))
else:
print('{:s}: -'.format(name))
gt_labels[i].append(
voc_detection_label_names.index(anno_elem['name']))
gt_difficults[i].append(anno_elem['difficult'])
gt_bboxes[i] = np.array(gt_bboxes[i])
gt_labels[i] = np.array(gt_labels[i])
gt_difficults[i] = np.array(gt_difficults[i], dtype=np.bool)
return gt_bboxes, gt_labels, gt_difficults, ids
if __name__ == '__main__':
base_dir = 'Main'
anno_fn = 'annotations_cache/annots.pkl'
gt_bboxes, gt_labels, gt_difficults, ids = read_gt_annos(anno_fn)
ids, bboxes, labels, confs = read_pascal_predictions(base_dir, ids)
metric = eval_detection_voc(bboxes,
labels,
confs,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=True)
labels = voc_detection_label_names[1:]
for i, label in enumerate(labels):
if i + 1 in metric:
print(label, metric[i + 1]['ap'])
print('>>>>>>>>>>>>>>>>>>>>>>>>>>>')
print metric['map']
nms_filter = nms(cls_region, cls_scores, 0.4)
per_pred_boxes.append(cls_region[nms_filter].numpy())
per_pred_scores.append(cls_scores[nms_filter].numpy())
per_pred_labels.append(np.full(len(nms_filter), i, dtype=np.int32))
total_pred_boxes.append(np.concatenate(per_pred_boxes, axis=0))
total_pred_scores.append(np.concatenate(per_pred_scores, axis=0))
total_pred_labels.append(np.concatenate(per_pred_labels, axis=0))
total_true_boxes.append(gt_boxex)
total_true_labels.append(gt_labels)
result = eval_detection_voc(
total_pred_boxes,
total_pred_labels,
total_pred_scores,
total_true_boxes,
total_true_labels,
iou_thresh=0.5,
use_07_metric=True,
)
write_log(log_file, f"Avg AP: {result['ap']}")
write_log(log_file, f"Avg mAP: {result['map']}")
cls_ap = []
y_pred = np.array(y_pred)
y_true = np.array(y_true)
for i in range(20):
cls_ap.append(average_precision_score(y_true[:,i], y_pred[:,i]))
write_log(log_file, f"Testset classify AP is {str(cls_ap)}")
write_log(log_file, f"Testset classify mAP is {str(sum(cls_ap)/20)}")
write_log(log_file, "-" * 30)
def evaluate(net, dataloader):
"""Evaluates network."""
with torch.no_grad():
net.eval()
total_pred_boxes = []
total_pred_scores = []
total_pred_labels = []
total_gt_boxes = []
total_gt_labels = []
for (
img_id,
img, # is it necessary
boxes,
scaled_imgs,
scaled_boxes,
scores,
gt_boxes,
gt_labels,
) in tqdm(dataloader, "Evaluation"):
combined_scores = torch.zeros(len(boxes), 20, dtype=torch.float32)
batch_scores = np2gpu(scores.numpy(), DEVICE)
for i, scaled_img in enumerate(scaled_imgs):
scaled_img = scaled_img.numpy()
tmp_scaled_boxes = scaled_boxes[i].numpy()
batch_imgs = np2gpu(scaled_img, DEVICE)
batch_boxes = np2gpu(tmp_scaled_boxes, DEVICE)
tmp_combined_scores = net(batch_imgs, batch_boxes,
batch_scores)
combined_scores += tmp_combined_scores.cpu()
combined_scores /= 10
gt_boxes = gt_boxes.numpy()
gt_labels = gt_labels.numpy()
batch_gt_boxes = np2gpu(gt_boxes, DEVICE)
batch_gt_labels = np2gpu(gt_labels, DEVICE)
batch_pred_boxes = []
batch_pred_scores = []
batch_pred_labels = []
for i in range(20):
region_scores = combined_scores[:, i]
score_mask = region_scores > 0
selected_scores = region_scores[score_mask]
selected_boxes = boxes[score_mask]
nms_mask = nms(selected_boxes, selected_scores, 0.4)
batch_pred_boxes.append(selected_boxes[nms_mask].cpu().numpy())
batch_pred_scores.append(
selected_scores[nms_mask].cpu().numpy())
batch_pred_labels.append(
np.full(len(nms_mask), i, dtype=np.int32))
total_pred_boxes.append(np.concatenate(batch_pred_boxes, axis=0))
total_pred_scores.append(np.concatenate(batch_pred_scores, axis=0))
total_pred_labels.append(np.concatenate(batch_pred_labels, axis=0))
total_gt_boxes.append(batch_gt_boxes[0].cpu().numpy())
total_gt_labels.append(batch_gt_labels[0].cpu().numpy())
result = eval_detection_voc(
total_pred_boxes,
total_pred_labels,
total_pred_scores,
total_gt_boxes,
total_gt_labels,
iou_thresh=0.5,
use_07_metric=True,
)
tqdm.write(f"Avg AP: {result['ap']}")
tqdm.write(f"Avg mAP: {result['map']}")
net.train()
def evaluate(net, dataloader, numADL, epoch):
"""Evaluates network."""
with torch.no_grad():
net.eval()
total_pred_boxes = []
total_pred_scores = []
total_pred_labels = []
total_gt_boxes = []
total_gt_labels = []
count = 0
for (
img_id,
img, # is it necessary
boxes,
scaled_imgs,
scaled_boxes,
scores,
gt_boxes,
gt_labels,
) in tqdm(dataloader, "Evaluation"):
count += 1
# pdb.set_trace()
# img_id = dataloader.dataset[336][0]
# img = dataloader.dataset[336][1]
# boxes = dataloader.dataset[336][2]
# scaled_imgs = dataloader.dataset[336][3]
# scaled_boxes = dataloader.dataset[336][4]
# scores = dataloader.dataset[336][5]
# gt_boxes = dataloader.dataset[336][0]
# gt_labels = dataloader.dataset[336][0]
combined_scores = torch.zeros(len(boxes), 20, dtype=torch.float32)
batch_scores = np2gpu(scores.numpy(), DEVICE)
# total_pred_boxes = []
# total_pred_scores = []
# total_pred_labels = []
# total_gt_boxes = []
# total_gt_labels = []
# if count >= 336:
# print('stop')
# pdb.set_trace()
for i, scaled_img in enumerate(scaled_imgs):
# print(i)
# scaled_img = scaled_img.numpy()
# tmp_scaled_boxes = scaled_boxes[i].numpy()
# batch_imgs = np2gpu(scaled_img, DEVICE)
# batch_boxes = np2gpu(tmp_scaled_boxes, DEVICE)
batch_imgs = scaled_img.unsqueeze(0).cuda()
batch_boxes = scaled_boxes[i].unsqueeze(0).cuda()
# tmp_combined_scores = net(batch_imgs, batch_boxes, batch_scores, gt_labels)
tmp_combined_scores = net.test(batch_imgs, batch_boxes,
batch_scores, gt_labels)
# tmp_combined_scores, _ = net(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch)
# tmp_combined_scores, _ = net.test(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch)
# tmp_combined_scores, _ = net.test_withD(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch)
# _, _, tmp_combined_scores = net(batch_imgs, batch_boxes, batch_scores, gt_labels.unsqueeze(0))
# pdb.set_trace()
combined_scores += tmp_combined_scores.cpu()
combined_scores /= 10
# gt_boxes = gt_boxes.numpy()
# gt_labels = gt_labels.numpy()
# batch_gt_boxes = np2gpu(gt_boxes, DEVICE)
# batch_gt_labels = np2gpu(gt_labels, DEVICE)
# pdb.set_trace()
batch_gt_labels = gt_labels.unsqueeze(0).cuda()
batch_gt_boxes = gt_boxes.unsqueeze(0).cuda()
batch_pred_boxes = []
batch_pred_scores = []
batch_pred_labels = []
for i in range(20):
# if i == 14
region_scores = combined_scores[:, i]
score_mask = region_scores > 0
selected_scores = region_scores[score_mask]
selected_boxes = boxes[score_mask]
nms_mask = nms(selected_boxes, selected_scores, 0.4)
# pdb.set_trace()
batch_pred_boxes.append(selected_boxes[nms_mask].cpu().numpy())
batch_pred_scores.append(
selected_scores[nms_mask].cpu().numpy())
batch_pred_labels.append(
np.full(len(nms_mask), i, dtype=np.int32))
total_pred_boxes.append(np.concatenate(batch_pred_boxes, axis=0))
total_pred_scores.append(np.concatenate(batch_pred_scores, axis=0))
total_pred_labels.append(np.concatenate(batch_pred_labels, axis=0))
total_gt_boxes.append(batch_gt_boxes[0].cpu().numpy())
total_gt_labels.append(batch_gt_labels[0].cpu().numpy())
# pdb.set_trace()
result = eval_detection_voc(total_pred_boxes,
total_pred_labels,
total_pred_scores,
total_gt_boxes,
total_gt_labels,
iou_thresh=0.5,
use_07_metric=True)
# print(result['map'])
# print(result['ap'])
tqdm.write(f"Avg AP: {result['ap']}")
tqdm.write(f"Avg mAP: {result['map']}")
net.train()
