def __init__(self, image_dir, anno_file):
#pass
if logger:
# logger.info('Creating: {}'.format(name))
logger.info('creating......image_oath:{},anno_file:{}'.format(
image_dir, anno_file))
self.image_directory = image_dir
self.image_prefix = ''
self.COCO = COCO(anno_file)
self.debug_timer = Timer()
#Set up dataset classes
category_ids = self.COCO.getCatIds()
categories = [c['name'] for c in self.COCO.loadCats(category_ids)]
self.category_to_id_map = dict(zip(categories, category_ids))
self.classes = ['__background__'] + categories
self.num_classes = len(self.classes)
self.json_category_id_to_contiguous_id = {
v: i + 1
for i, v in enumerate(self.COCO.getCatIds())
}
self.contiguous_category_id_to_json_id = {
v: k
for k, v in self.json_category_id_to_contiguous_id.items()
}
def evaluate_boxes(self, all_boxes, output_dir):
res_file = os.path.join(output_dir,
'bbox_' + self.dataset.name + '_results.json')
results = []
for cls_ind, cls in enumerate(self.dataset.classes):
if cls == '__background__':
continue
if cls_ind >= len(all_boxes):
break
cat_id = self.dataset.category_to_id_map[cls]
results.extend(
self.bbox_results_one_category(all_boxes[cls_ind], cat_id))
if logger:
logger.info('Writing bbox results json to: {}'.format(
os.path.abspath(res_file)))
with open(res_file, 'w') as fid:
json.dump(results, fid)
coco_dt = self.dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(self.dataset.COCO, coco_dt, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
self.log_detection_eval_metrics(
coco_eval, os.path.join(output_dir, 'detection_results.txt'))
return coco_eval.stats
def filter_for_training(roidb):
"""Remove roidb entries that have no usable RoIs based on config settings.
"""
def is_valid(entry):
# Valid images have:
# (1) At least one foreground RoI OR
# (2) At least one background RoI
overlaps = entry['max_overlaps']
# find boxes with sufficient overlap
fg_inds = np.where(overlaps >= config.train.fg_thresh)[0]
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((overlaps < config.train.bg_thresh_hi)
& (overlaps >= config.train.bg_thresh_lo))[0]
# image is only valid if such boxes exist
valid = len(fg_inds) > 0 or len(bg_inds) > 0
# if cfg.MODEL.KEYPOINTS_ON:
# # If we're training for keypoints, exclude images with no keypoints
# valid = valid and entry['has_visible_keypoints']
return valid
num = len(roidb)
filtered_roidb = [entry for entry in roidb if is_valid(entry)]
num_after = len(filtered_roidb)
if logger:
logger.info('Filtered {} roidb entries: {} -> {}'.format(
num - num_after, num, num_after))
return filtered_roidb
def _add_proposals_from_file(self, roidb, proposal_file, min_proposal_size,
top_k, crowd_thresh):
"""Add proposals from a proposals file to an roidb."""
if logger:
logger.info('Loading proposals from: {}'.format(proposal_file))
with open(proposal_file, 'rb') as f:
proposals = pickle.load(f)
id_field = 'indexes' if 'indexes' in proposals else 'ids' # compat fix
_sort_proposals(proposals, id_field)
box_list = []
for i, entry in enumerate(roidb):
if i % 2500 == 0:
if logger:
logger.info(' {:d}/{:d}'.format(i + 1, len(roidb)))
boxes = proposals['boxes'][i]
# Sanity check that these boxes are for the correct image id
assert entry['id'] == proposals[id_field][i]
# Remove duplicate boxes and very small boxes and then take top k
boxes = box_utils.clip_boxes_to_image(boxes, entry['height'],
entry['width'])
keep = box_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = box_utils.filter_boxes(boxes, min_proposal_size)
boxes = boxes[keep, :]
if top_k > 0:
boxes = boxes[:top_k, :]
box_list.append(boxes)
_merge_proposal_boxes_into_roidb(roidb, box_list)
if crowd_thresh > 0:
_filter_crowd_proposals(roidb, crowd_thresh)
def evaluate_masks(
self,
all_boxes,
all_segms,
output_dir,
):
res_file = os.path.join(
output_dir, 'segmentations_' + self.dataset.name + '_results.json'
)
results = []
for cls_ind, cls in enumerate(self.dataset.classes):
if cls == '__background__':
continue
if cls_ind >= len(all_boxes):
break
cat_id = self.dataset.category_to_id_map[cls]
results.extend(self.segms_results_one_category(all_boxes[cls_ind], all_segms[cls_ind], cat_id))
if logger:
logger.info(
'Writing segmentation results json to: {}'.format(
os.path.abspath(res_file)))
with open(res_file, 'w') as fid:
json.dump(results, fid)
coco_dt = self.dataset.COCO.loadRes(str(res_file))
coco_eval = COCOeval(self.dataset.COCO, coco_dt, 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
self.log_detection_eval_metrics(coco_eval, os.path.join(output_dir, 'detection_results.txt'))
# eval_file = os.path.join(output_dir, 'segmentation_results.pkl')
# pickle.dump(coco_eval, open(eval_file, 'wb'), pickle.HIGHEST_PROTOCOL)
# if logger:
# logger.info('Wrote json eval results to: {}'.format(eval_file))
# Optionally cleanup results json file
# os.remove(res_file)
return coco_eval
def __init__(self,
params,
lr=required,
momentum=0,
dampening=0,
weight_decay=0,
nesterov=False):
defaults = dict(lr=lr,
momentum=momentum,
dampening=dampening,
weight_decay=weight_decay,
nesterov=nesterov)
if logger:
logger.info(defaults)
if nesterov and (momentum <= 0 or dampening != 0):
raise ValueError(
"Nesterov momentum requires a momentum and zero dampening")
assert dampening == 0, "not implemented"
super(SGD, self).__init__(params, defaults)
def log_detection_eval_metrics(self, coco_eval, log_file):
def _get_thr_ind(coco_eval, thr):
ind = np.where((coco_eval.params.iouThrs > thr - 1e-5)
& (coco_eval.params.iouThrs < thr + 1e-5))[0][0]
iou_thr = coco_eval.params.iouThrs[ind]
assert np.isclose(iou_thr, thr)
return ind
class tee:
def __init__(self, *files):
self.files = files
def write(self, obj):
for f in self.files:
f.write(obj)
stdout = sys.stdout
sys.stdout = tee(sys.stdout, open(log_file, 'w'))
IoU_lo_thresh = 0.5
for IoU_hi_thresh in [0.95, 0.5]:
ind_lo = _get_thr_ind(coco_eval, IoU_lo_thresh)
ind_hi = _get_thr_ind(coco_eval, IoU_hi_thresh)
# precision has dims (iou, recall, cls, area range, max dets)
# area range index 0: all area ranges
# max dets index 2: 100 per image
precision = coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, :,
0, 2]
ap_default = np.mean(precision[precision > -1])
if logger:
logger.info(
'~~~~ Mean and per-category AP @ IoU=[{:.2f},{:.2f}] ~~~~'.
format(IoU_lo_thresh, IoU_hi_thresh))
for cls_ind, cls in enumerate(self.dataset.classes):
if cls == '__background__':
continue
# minus 1 because of __background__
precision = coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :,
cls_ind - 1, 0, 2]
ap = np.mean(precision[precision > -1])
if logger:
logger.info('{:.3f}'.format(ap))
if logger:
logger.info('{:.3f}'.format(ap_default))
if logger:
logger.info('~~~~ Summary metrics ~~~~')
coco_eval.summarize()
sys.stdout = stdout
def pq_compute(gt_jsons, pred_jsons, gt_pans, pred_pans, categories):
start_time = time.time()
# from json and from numpy
gt_image_jsons = gt_jsons['images']
gt_jsons, pred_jsons = gt_jsons['annotations'], pred_jsons[
'annotations']
cpu_num = multiprocessing.cpu_count()
gt_jsons_split, pred_jsons_split = np.array_split(
gt_jsons, cpu_num), np.array_split(pred_jsons, cpu_num)
gt_pans_split, pred_pans_split = np.array_split(
gt_pans, cpu_num), np.array_split(pred_pans, cpu_num)
gt_image_jsons_split = np.array_split(gt_image_jsons, cpu_num)
workers = multiprocessing.Pool(processes=cpu_num)
processes = []
for proc_id, (gt_jsons_set, pred_jsons_set, gt_pans_set,
pred_pans_set, gt_image_jsons_set) in enumerate(
zip(gt_jsons_split, pred_jsons_split,
gt_pans_split, pred_pans_split,
gt_image_jsons_split)):
p = workers.apply_async(
BaseDataset._pq_compute_single_core,
(proc_id, gt_jsons_set, pred_jsons_set, gt_pans_set,
pred_pans_set, gt_image_jsons_set, categories))
processes.append(p)
workers.close()
workers.join()
pq_stat = PQStat()
for p in processes:
pq_stat += p.get()
metrics = [("All", None), ("Things", True), ("Stuff", False)]
results = {}
for name, isthing in metrics:
results[name], per_class_results = pq_stat.pq_average(
categories, isthing=isthing)
if name == 'All':
results['per_class'] = per_class_results
if logger:
logger.info("{:10s}| {:>5s} {:>5s} {:>5s} {:>5s}".format(
"", "PQ", "SQ", "RQ", "N"))
logger.info("-" * (10 + 7 * 4))
for name, _isthing in metrics:
logger.info(
"{:10s}| {:5.1f} {:5.1f} {:5.1f} {:5d}".format(
name, 100 * results[name]['pq'],
100 * results[name]['sq'],
100 * results[name]['rq'], results[name]['n']))
logger.info(
"{:4s}| {:>5s} {:>5s} {:>5s} {:>6s} {:>7s} {:>7s} {:>7s}".
format("IDX", "PQ", "SQ", "RQ", "IoU", "TP", "FP", "FN"))
for idx, result in results['per_class'].items():
logger.info(
"{:4d} | {:5.1f} {:5.1f} {:5.1f} {:6.1f} {:7d} {:7d} {:7d}"
.format(idx, 100 * result['pq'], 100 * result['sq'],
100 * result['rq'], result['iou'],
result['tp'], result['fp'], result['fn']))
t_delta = time.time() - start_time
print("Time elapsed: {:0.2f} seconds".format(t_delta))
return results
def __init__(self,
image_sets,
flip=False,
proposal_files=None,
phase='train',
result_path=''):
super(coco, self).__init__()
image_dirs = {
'train2014':
os.path.join(config.dataset.dataset_path, 'coco_train2014'),
'val2014':
os.path.join(config.dataset.dataset_path, 'coco_val2014'),
'minival2014':
os.path.join(config.dataset.dataset_path, 'coco_val2014'),
'valminusminival2014':
os.path.join(config.dataset.dataset_path, 'coco_val2014'),
'test2015':
os.path.join(config.dataset.dataset_path, 'coco_test2015'),
'test-dev2015':
os.path.join(config.dataset.dataset_path, 'coco_test2015'),
'train2017':
os.path.join(config.dataset.dataset_path, 'images', 'train2017'),
'val2017':
os.path.join(config.dataset.dataset_path, 'images', 'val2017'),
'test-dev2017':
os.path.join(config.dataset.dataset_path, 'images', 'test2017'),
}
anno_files = {
'train2014': 'instances_train2014.json',
'val2014': 'instances_val2014.json',
'minival2014': 'instances_minival2014.json',
'valminusminival2014': 'instances_valminusminival2014.json',
'test2015': 'image_info_test2015.json',
'test-dev2015': 'image_info_test-dev2015.json',
'train2017': 'instances_train2017.json',
'val2017': 'instances_val2017.json',
'test-dev2017': 'image_info_test-dev2017.json',
}
if image_sets[0] == 'test-dev2017':
self.panoptic_json_file = os.path.join(
config.dataset.dataset_path, 'annotations',
'image_info_test-dev2017.json')
else:
self.panoptic_json_file = os.path.join(
config.dataset.dataset_path, 'annotations',
'panoptic_val2017_stff.json')
self.panoptic_gt_folder = os.path.join(config.dataset.dataset_path,
'annotations',
'panoptic_val2017')
if proposal_files is None:
proposal_files = [None] * len(image_sets)
if phase == 'train' and len(image_sets) > 1:
# combine multiple datasets
roidbs = []
for image_set, proposal_file in zip(image_sets, proposal_files):
dataset = JsonDataset('coco_' + image_set,
image_dir=image_dirs[image_set],
anno_file=os.path.join(
config.dataset.dataset_path,
'annotations',
anno_files[image_set]))
roidb = dataset.get_roidb(
gt=True,
proposal_file=proposal_file,
crowd_filter_thresh=config.train.crowd_filter_thresh)
if flip:
if logger:
logger.info(
'Appending horizontally-flipped training examples...'
)
extend_with_flipped_entries(roidb, dataset)
roidbs.append(roidb)
roidb = roidbs[0]
for r in roidbs[1:]:
roidb.extend(r)
roidb = filter_for_training(roidb)
add_bbox_regression_targets(roidb)
else:
assert len(image_sets) == 1
self.dataset = JsonDataset('coco_' + image_sets[0],
image_dir=image_dirs[image_sets[0]],
anno_file=os.path.join(
config.dataset.dataset_path,
'annotations',
anno_files[image_sets[0]]))
roidb = self.dataset.get_roidb(
gt=True,
proposal_file=proposal_files[0],
crowd_filter_thresh=config.train.crowd_filter_thresh
if phase != 'test' else 0)
if flip:
if logger:
logger.info(
'Appending horizontally-flipped training examples...')
extend_with_flipped_entries(roidb, self.dataset)
if phase != 'test':
roidb = filter_for_training(roidb)
add_bbox_regression_targets(roidb)
self.roidb = roidb
self.phase = phase
self.flip = flip
self.result_path = result_path
self.num_classes = 81
def evaluate_ssegs(self, pred_segmentations, res_file_folder):
self.write_segmentation_result(pred_segmentations, res_file_folder)
confusion_matrix = np.zeros(
(config.dataset.num_seg_classes, config.dataset.num_seg_classes))
for i, roidb in enumerate(self.roidb):
seg_gt = np.array(
Image.open(self.roidb[i]['image'].replace(
'images', 'annotations').replace(
'train2017',
'panoptic_train2017_semantic_trainid_stff').replace(
'val2017',
'panoptic_val2017_semantic_trainid_stff').replace(
'jpg', 'png'))).astype(np.float32)
seg_pathes = os.path.split(roidb['image'])
res_image_name = seg_pathes[-1]
res_save_path = os.path.join(res_file_folder,
res_image_name + '.png')
seg_pred = Image.open(res_save_path)
seg_pred = np.array(
seg_pred.resize((seg_gt.shape[1], seg_gt.shape[0]),
Image.NEAREST))
ignore_index = seg_gt != 255
seg_gt = seg_gt[ignore_index]
seg_pred = seg_pred[ignore_index]
confusion_matrix += self.get_confusion_matrix(
seg_gt, seg_pred, config.dataset.num_seg_classes)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IU = IU_array.mean()
evaluation_results = {
'meanIU': mean_IU,
'IU_array': IU_array,
'confusion_matrix': confusion_matrix
}
def convert_confusion_matrix(confusion_matrix):
cls_sum = confusion_matrix.sum(axis=1)
confusion_matrix = confusion_matrix / cls_sum.reshape((-1, 1))
return confusion_matrix
logger.info('evaluate segmentation:')
meanIU = evaluation_results['meanIU']
IU_array = evaluation_results['IU_array']
confusion_matrix = convert_confusion_matrix(
evaluation_results['confusion_matrix'])
logger.info('IU_array:')
for i in range(len(IU_array)):
logger.info('%.5f' % IU_array[i])
logger.info('meanIU:%.5f' % meanIU)
np.set_printoptions(precision=3, suppress=True, linewidth=200)
import re
confusion_matrix = re.sub(
'[\[\]]', '', np.array2string(confusion_matrix, separator='\t'))
logger.info('confusion_matrix:')
logger.info(confusion_matrix)
def __init__(self,
image_sets,
flip=False,
proposal_files=None,
phase='train',
result_path=''):
super(Cityscapes, self).__init__()
self.image_dirs = {
'train': os.path.join(config.dataset.dataset_path, 'images'),
'val': os.path.join(config.dataset.dataset_path, 'images'),
'test': os.path.join(config.dataset.dataset_path, 'images'),
'train_extra': os.path.join(config.dataset.dataset_path, 'images'),
'debug': os.path.join(config.dataset.dataset_path, 'images'),
}
self.anno_files = {
'train': 'instancesonly_gtFine_train.json',
'val': 'instancesonly_gtFine_val.json',
'test': 'image_info_test.json',
'train_extra': 'instancesonly_gtCoarse_train_extra.json',
'debug': 'instancesonly_gtFine_debug.json',
}
self.panoptic_json_file = os.path.join(config.dataset.dataset_path,
'annotations',
'cityscapes_fine_val.json')
self.panoptic_gt_folder = 'data/cityscapes/panoptic'
self.flip = flip
self.result_path = result_path
self.num_classes = 9
self.phase = phase
self.image_sets = image_sets
if image_sets[0] == 'demoVideo':
assert len(image_sets) == 1
assert phase == 'test'
im_path = [
_.strip()
for _ in open('data/cityscapes/split/demoVideo_img.txt',
'r').readlines()
]
self.roidb = [{'image': _, 'flipped': False} for _ in im_path]
return
if proposal_files is None:
proposal_files = [None] * len(image_sets)
if phase == 'train' and len(image_sets) > 1:
# combine multiple datasets
roidbs = []
for image_set, proposal_file in zip(image_sets, proposal_files):
dataset = JsonDataset('cityscapes_' + image_set,
image_dir=self.image_dirs[image_set],
anno_file=os.path.join(
config.dataset.dataset_path,
'annotations',
self.anno_files[image_set]))
roidb = dataset.get_roidb(
gt=True,
proposal_file=proposal_file,
crowd_filter_thresh=config.train.crowd_filter_thresh)
if flip:
if logger:
logger.info(
'Appending horizontally-flipped training examples...'
)
extend_with_flipped_entries(roidb, dataset)
roidbs.append(roidb)
roidb = roidbs[0]
for r in roidbs[1:]:
roidb.extend(r)
roidb = filter_for_training(roidb)
add_bbox_regression_targets(roidb)
else:
assert len(image_sets) == 1
self.dataset = JsonDataset(
'cityscapes_' + image_sets[0],
image_dir=self.image_dirs[image_sets[0]],
anno_file=os.path.join(config.dataset.dataset_path,
'annotations',
self.anno_files[image_sets[0]]))
roidb = self.dataset.get_roidb(
gt=True,
proposal_file=proposal_files[0],
crowd_filter_thresh=config.train.crowd_filter_thresh
if phase != 'test' else 0)
if flip:
if logger:
logger.info(
'Appending horizontally-flipped training examples...')
extend_with_flipped_entries(roidb, self.dataset)
if phase != 'test':
roidb = filter_for_training(roidb)
add_bbox_regression_targets(roidb)
self.roidb = roidb
