def evaluateByJsp(resJsp,
gtJsp,
log=True,
method=None,
levels=['averaged', 'easy', 'medium', 'hard']):
if method is None:
method = basename(dirname(dirname(dirname(resJsp))))
# method=basename(dirname(resJsp))
resTable = defaultdict(lambda: {})
resJs = loadjson(resJsp)
gtJs = loadjson(gtJsp)
if 'averaged' in levels:
level = 'averaged'
row = evaluateByJson(
resJs,
gtJs,
)
row['method'] = method
row['level'] = level
resTable[level] = dict(row)
tree - row
for level in filter(lambda x: x in levels, ['easy', 'medium', 'hard']):
coco = loadjson(gtJsp)
coco['images'] = [d for d in coco['images'] if d['level'] == level]
imgIds = [d['id'] for d in coco['images']]
coco['annotations'] = [
bb for bb in coco['annotations'] if bb['image_id'] in imgIds
]
resJs = loadjson(resJsp)
row = evaluateByJson(
resJs,
coco,
)
row['method'] = method
row['level'] = level
resTable[level] = dict(row)
tree - row
resdir = dirname(resJsp)
savejson(resTable, pathjoin(resdir, 'resTable.json'))
return resTable
def cover2rpc(sourceFile, targetFile, reLevel=False):
from boxx import openwrite, openread, df2dicts, zip2
sjs = boxx.loadjson(sourceFile)
tjs = boxx.loadjson(targetFile)
openwrite(openread(sourceFile), sourceFile + '.before.cover2rpc.bak')
if reLevel:
imgdf = pd.DataFrame(sjs["images"])
anndf = pd.DataFrame(sjs["annotations"])
imgid2len = anndf.groupby('image_id').apply(lambda sdf: [
len(sdf),
len(set(sdf.category_id)),
])
imgdf['instance_num'] = imgdf.id.apply(lambda i: imgid2len.loc[i][0])
imgdf['class_num'] = imgdf.id.apply(lambda i: imgid2len.loc[i][1])
imgdf = imgdf.sort_values(['instance_num'])
n = len(imgdf)
indexDic = dict(
zip2(range(n // 3), ['easy'] * n) +
zip2(range(n // 3, 2 * n // 3), ['medium'] * n) +
zip2(range(2 * n // 3, n), ['hard'] * n))
imgdf['_ind'] = range(n)
imgdf['level'] = imgdf._ind.apply(lambda i: indexDic[i])
imgdf.pop('_ind')
# imgdf.iloc[:n//3].level = 'easy'
# imgdf.iloc[n//3:n*2//3].level = 'medium'
# imgdf.iloc[n*2//3:].level = 'hard'
assert all(imgdf.iloc[n * 2 // 3:].level == 'hard'), imgdf.iloc[-1]
imgdf.pop('instance_num')
imgdf.pop('class_num')
dicts = df2dicts(imgdf)
tjs['images'] = dicts
else:
tjs['images'] = sjs["images"]
tjs['annotations'] = sjs["annotations"]
return boxx.savejson(tjs, sourceFile)
def get_json_items_df(json_file_path):
json = boxx.loadjson(json_file_path)
json_items_df = pd.DataFrame(json['__raw_Chinese_name_df'])
json_items_df = json_items_df[[
'sku_name', 'category_id', 'sku_class', 'code', 'shelf', 'num', 'name',
'clas', 'known', 'ind'
]]
return json_items_df
def get_json_images_df(json_file_path):
json = boxx.loadjson(json_file_path)
json_images_df = pd.DataFrame(json['images'])
json_images_df = json_images_df.rename(columns={
'height': 'image_height',
'width': 'image_width',
'id': 'image_id'
})
json_images_df = json_images_df[[
'file_name', 'image_width', 'image_height', 'image_id'
]]
return json_images_df
def get_json_annotations_df(json_file_path):
json = boxx.loadjson(json_file_path)
json_annotations_df = pd.DataFrame(json['annotations'])
json_annotations_df = json_annotations_df.rename(
columns={
'area': 'image_area',
'bbox': 'image_bbox',
'point_xy': 'image_point_xy',
'segmentation': 'image_segmentation',
'iscrowd': 'is_crowded'
})
json_annotations_df = json_annotations_df[[
'image_id', 'image_area', 'image_bbox', 'image_point_xy',
'image_segmentation', 'is_crowded', 'category_id'
]]
return json_annotations_df
print(resJsp)
thre = .784321 # mix 11
thre = 0.84 # testAsTrain
diff = 'easy'
#diff = 'medium'
#diff = 'hard'
diff = 'all'
getWrong = True
#getWrong = False
xuanMai = [i + 1 for i in [135, 136, 137]]
if 1:
valJsp = '../checkout_submission_tools/tmp_file_json/instances_test2017.json'
coco = loadjson(valJsp)
imgds = coco['images']
imgdf = pd.DataFrame(imgds)
imgdf = imgdf.set_index('id')
imgdf['id'] = imgdf.index
if diff != 'all':
imgdf = imgdf[imgdf['level'] == diff]
imgIds = set(imgdf.id)
gtanns = coco['annotations']
gtdf = pd.DataFrame(gtanns)
gtdf = gtdf[gtdf.image_id.isin(imgIds)]
# gtdf['level'] = gtdf.image_id.apply(lambda idd: imgdf.loc[idd]['level'])
gtdf['fname'] = gtdf.image_id.apply(
def rpc_evaluation(
dataset,
predictions,
output_folder,
generate_pseudo_labels=False,
iteration=-1,
threshold=THRESHOLD,
use_ground_truth=False, # use ground truth to select pseudo labels
**_):
threshold = 0.9995 if threshold >= 1 else threshold
logger = logging.getLogger("maskrcnn_benchmark.inference")
if generate_pseudo_labels:
logger.info('Use ground truth: {}'.format(use_ground_truth))
pred_boxlists = []
pred_boxlists_with_density = []
annotations = []
density_correct = 0
box_correct = 0
mae = 0 # MEAN ABSOLUTE ERROR
has_density_map = predictions[0].has_field('density')
num_density_classes = 1
if has_density_map:
num_density_classes = predictions[0].get_field('density').shape[-1]
logger.info('Density category: {}'.format(num_density_classes))
for image_id, prediction in tqdm(enumerate(predictions)):
img_info = dataset.get_img_info(image_id)
image_width = img_info["width"]
image_height = img_info["height"]
prediction = prediction.resize((image_width, image_height))
bboxes = prediction.bbox.tolist()
labels = prediction.get_field("labels").tolist()
scores = prediction.get_field("scores").tolist()
# -----------------------------------------------#
# -----------------Pseudo Label------------------#
# -----------------------------------------------#
gt_density_cat_counts = np.zeros((num_density_classes, ),
dtype=np.int32)
gt_all_cat_counts = np.zeros((NUM_CLASSES, ), dtype=np.int32)
pred_density_cat_counts = np.zeros((num_density_classes, ),
dtype=np.int32)
# density = 0.0
if has_density_map:
pred_density_cat_counts = prediction.get_field('density').numpy()
pred_density_cat_map = prediction.get_field('density_map').numpy()
pred_density_cat_counts = np.round(pred_density_cat_counts).astype(
np.int32)
ann = dataset.get_annotation(img_info['id'])
for category, x, y, w, h in ann:
density_category = rpc_category_to_super_category(
category, num_classes=num_density_classes)
gt_all_cat_counts[category] += 1
gt_density_cat_counts[density_category] += 1
is_correct = np.all(
gt_density_cat_counts == pred_density_cat_counts)
if is_correct:
density_correct += 1
else:
mae += np.sum(
np.abs(gt_density_cat_counts - pred_density_cat_counts))
box_density_cat_counts = np.zeros((num_density_classes, ),
dtype=np.int32)
box_all_cat_counts = np.zeros((NUM_CLASSES, ), dtype=np.int32)
if generate_pseudo_labels and has_density_map:
image_result = {
'bbox': [],
'width': image_width,
'height': image_height,
'id': img_info['id'],
'file_name': img_info['file_name'],
}
for i in range(len(prediction)):
score = scores[i]
if score > threshold:
box = bboxes[i]
label = labels[i]
density_category = rpc_category_to_super_category(
label, num_density_classes)
box_all_cat_counts[label] += 1
box_density_cat_counts[density_category] += 1
x, y, width, height = box[0], box[
1], box[2] - box[0], box[3] - box[1]
image_result['bbox'].append((label, x, y, width, height))
if use_ground_truth:
is_valid = np.all(box_all_cat_counts == gt_all_cat_counts)
else:
is_valid = np.all(
box_density_cat_counts == pred_density_cat_counts)
if is_valid:
annotations.append(image_result)
is_box_correct = np.all(
box_all_cat_counts == gt_all_cat_counts)
if is_box_correct:
box_correct += 1
# -----------------------------------------------#
# -----------------------------------------------#
# -----------------------------------------------#
for i in range(len(prediction)):
score = scores[i]
box = bboxes[i]
label = labels[i]
x, y, width, height = box[0], box[
1], box[2] - box[0], box[3] - box[1]
pred_boxlists.append({
"image_id":
img_info['id'],
"category_id":
int(label),
"bbox": [float(k) for k in [x, y, width, height]],
"score":
float(score),
})
pred_boxlists_with_density.append({
"image_id":
img_info['id'],
"category_id":
int(label),
"bbox": [float(k) for k in [x, y, width, height]],
"score":
float(score),
"density_map":
pred_density_cat_map
})
if has_density_map:
logger.info('Density Ratio: {:.3f}'.format(density_correct /
len(predictions)))
logger.info('Density MAE : {:.3f} '.format(mae / len(predictions)))
if generate_pseudo_labels:
if len(annotations) == 0:
logger.info('No annotations are selected.')
else:
logger.info(
'Select Ratio: {:.3f} ({}/{}, {:.5f} Threshold)'.format(
box_correct / len(annotations), box_correct,
len(annotations), threshold))
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if len(pred_boxlists) == 0:
logger.info('Nothing detected.')
with open(
os.path.join(output_folder,
'result_{}.txt'.format(time_stamp)), 'w') as fid:
fid.write('Nothing detected.')
return dict(metrics={})
if generate_pseudo_labels:
logger.info('Pseudo-Labeling: {}'.format(len(annotations)))
with open(os.path.join(output_folder, 'pseudo_labeling.json'),
'w') as fid:
json.dump(annotations, fid)
save_path = os.path.join(output_folder, 'bbox_results.json')
with open(save_path, 'w') as fid:
json.dump(pred_boxlists, fid)
mmcv.dump(pred_boxlists_with_density,
os.path.join(output_folder, 'bbox_results_with_density.pkl'))
res_js = boxx.loadjson(save_path)
ann_js = boxx.loadjson(dataset.ann_file)
result = rpctool.evaluate(res_js, ann_js)
logger.info(result)
result_str = str(result)
if iteration > 0:
filename = os.path.join(output_folder,
'result_{:07d}.txt'.format(iteration))
else:
filename = os.path.join(output_folder,
'result_{}.txt'.format(time_stamp))
if has_density_map:
result_str += '\n' + 'Ratio: {:.3f}, '.format(
density_correct / len(predictions)) + 'MAE: {:.3f} '.format(
mae / len(predictions))
with open(filename, 'w') as fid:
fid.write(result_str)
best_cAcc = check_best_result(output_folder, result, result_str, filename)
logger.info('Best cAcc: {}%'.format(best_cAcc))
metrics = {
'cAcc': {
'averaged': get_cAcc(result, 'averaged'),
'hard': get_cAcc(result, 'hard'),
'medium': get_cAcc(result, 'medium'),
'easy': get_cAcc(result, 'easy'),
}
}
if has_density_map:
metrics.update({
'Ratio': density_correct / len(predictions),
'MAE': mae / len(predictions),
})
eval_result = dict(metrics=metrics)
return eval_result
def rpc_evaluation(dataset, predictions, output_folder, generate_pseudo_labels=False, iteration=-1, **_):
logger = logging.getLogger("maskrcnn_benchmark.inference")
pred_boxlists = []
annotations = []
correct = 0
mae = 0 # mean absolute error
has_density_map = predictions[0].has_field('density_map')
for image_id, prediction in tqdm(enumerate(predictions)):
img_info = dataset.get_img_info(image_id)
image_width = img_info["width"]
image_height = img_info["height"]
prediction = prediction.resize((image_width, image_height))
bboxes = prediction.bbox.numpy()
labels = prediction.get_field("labels").numpy()
scores = prediction.get_field("scores").numpy()
# -----------------------------------------------#
# -----------------Pseudo Label------------------#
# -----------------------------------------------#
density = 0.0
if has_density_map:
ann = dataset.get_annotation(img_info['id'])
density_map = prediction.get_field('density_map').numpy()
density = density_map.sum()
if round(density) == len(ann):
correct += 1
mae += abs(density - len(ann))
if generate_pseudo_labels and has_density_map:
image_result = {
'bbox': [],
'width': image_width,
'height': image_height,
'id': img_info['id'],
'file_name': img_info['file_name'],
}
for i in range(len(prediction)):
score = scores[i]
box = bboxes[i]
label = labels[i]
if score > 0.95:
x, y, width, height = float(box[0]), float(box[1]), float(box[2] - box[0]), float(box[3] - box[1])
image_result['bbox'].append(
(int(label), x, y, width, height)
)
if len(image_result['bbox']) >= 3 and len(image_result['bbox']) == round(density):
annotations.append(image_result)
# -----------------------------------------------#
# -----------------------------------------------#
# -----------------------------------------------#
for i in range(len(prediction)):
score = scores[i]
box = bboxes[i]
label = labels[i]
x, y, width, height = box[0], box[1], box[2] - box[0], box[3] - box[1]
pred_boxlists.append({
"image_id": img_info['id'],
"category_id": int(label),
"bbox": [float(k) for k in [x, y, width, height]],
"score": float(score),
})
if has_density_map:
logger.info('Ratio: {}'.format(correct / len(predictions)))
logger.info('MAE: {:.3f} '.format(mae / len(predictions)))
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if len(pred_boxlists) == 0:
logger.info('Nothing detected.')
with open(os.path.join(output_folder, 'result_{}.txt'.format(time_stamp)), 'w') as fid:
fid.write('Nothing detected.')
return 'Nothing detected.'
if generate_pseudo_labels:
logger.info('Pseudo-Labeling: {}'.format(len(annotations)))
with open(os.path.join(output_folder, 'pseudo_labeling.json'), 'w') as fid:
json.dump(annotations, fid)
save_path = os.path.join(output_folder, 'bbox_results.json')
with open(save_path, 'w') as fid:
json.dump(pred_boxlists, fid)
res_js = boxx.loadjson(save_path)
ann_js = boxx.loadjson(dataset.ann_file)
result = rpctool.evaluate(res_js, ann_js)
logger.info(result)
result_str = str(result)
if iteration > 0:
filename = os.path.join(output_folder, 'result_{:07d}.txt'.format(iteration))
else:
filename = os.path.join(output_folder, 'result_{}.txt'.format(time_stamp))
if has_density_map:
result_str += '\n' + 'Ratio: {}, '.format(correct / len(predictions)) + 'MAE: {:.3f} '.format(mae / len(predictions))
with open(filename, 'w') as fid:
fid.write(result_str)
best_cAcc = check_best_result(output_folder, result, result_str, filename)
logger.info('Best cAcc: {}%'.format(best_cAcc))
metrics = {
'cAcc': {
'averaged': get_cAcc(result, 'averaged'),
'hard': get_cAcc(result, 'hard'),
'medium': get_cAcc(result, 'medium'),
'easy': get_cAcc(result, 'easy'),
}
}
if has_density_map:
metrics.update({
'Ratio': correct / len(predictions),
'MAE': mae / len(predictions),
})
eval_result = dict(metrics=metrics)
return eval_result
default=None,
type=lambda s: s and s.replace(" ", '').split(','),
help='Which different levels, default is "easy,medium,hard,averaged"',
)
parser.add_argument(
"--vis",
action='store_true',
help="visualization after evaluate",
)
parser.add_argument(
"--cn",
action='store_true',
help="Use raw Chinese class name, to instead of English name",
)
if __name__ == '__main__':
args = parser.parse_args()
resJs = boxx.loadjson(args.resFile)
annJs = boxx.loadjson(args.annFile)
skudf = get_skudf(annJs)
md = evaluate(resJs,
annJs,
mmap=args.mmap,
method=args.method,
levels=args.levels)
print(
'''\nYou could submit this markdown result to RPC-Leaderboard by new a issue here:
https://github.com/RPC-Dataset/RPC-Leaderboard/issues''')
print("\n## %s result on RPC-Dataset" % args.method)
print(md)
